diff --git "a/wandb/run-20220830_110431-yvlr8ud4/logs/debug-internal.log" "b/wandb/run-20220830_110431-yvlr8ud4/logs/debug-internal.log" --- "a/wandb/run-20220830_110431-yvlr8ud4/logs/debug-internal.log" +++ "b/wandb/run-20220830_110431-yvlr8ud4/logs/debug-internal.log" @@ -56880,3 +56880,10937 @@ 2022-08-30 18:55:12,370 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats 2022-08-30 18:55:19,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status 2022-08-30 18:55:19,115 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 18:55:30,892 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:55:32,893 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:55:34,336 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 18:55:34,337 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 18:55:34,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:34,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:34,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:34,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:34,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:34,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:34,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:34,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:34,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:34,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:34,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:34,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:34,713 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:34,713 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:34,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:34,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:34,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:34,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:34,844 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:34,845 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:34,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:34,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:34,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:34,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:34,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:34,858 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:34,860 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:34,860 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:34,862 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:34,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:34,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:34,864 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:34,933 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:34,934 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:34,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:34,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:34,940 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:34,941 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:34,942 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:34,943 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:34,952 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:34,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:34,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:34,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:34,957 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:34,957 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:34,958 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:34,959 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:38,895 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:55:42,446 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 18:55:42,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:42,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:42,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:42,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:42,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:42,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:42,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:42,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:42,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:42,664 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:42,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:42,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:42,668 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:42,668 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:42,669 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:42,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:42,897 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:55:46,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:46,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:46,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:46,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:46,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:46,661 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:46,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:46,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:46,668 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:46,668 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:46,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:46,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:46,672 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:46,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:46,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:46,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:46,899 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:55:48,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:48,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:48,586 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:48,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:48,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:48,588 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:48,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:48,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:48,595 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:48,596 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:48,597 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:48,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:48,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:48,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:48,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:48,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:48,676 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:48,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:48,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:48,682 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:48,682 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:48,683 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:48,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:48,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:48,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:48,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:48,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:48,693 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:48,694 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:48,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:48,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:48,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:49,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 18:55:49,574 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 18:55:50,469 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:50,470 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:50,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:50,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:50,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:50,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:50,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:50,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:50,482 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:50,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:50,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:50,485 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:50,486 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:50,487 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:50,487 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:50,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:50,900 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:55:52,252 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:52,252 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:52,256 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:52,256 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:52,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:52,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:52,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:52,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:52,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:52,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:52,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:52,267 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:52,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:52,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:52,270 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:52,270 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:54,902 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:55:55,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:55,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:55,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:55,165 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:55,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:55,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:55,167 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:55,168 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:55,172 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:55,173 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:55,175 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:55,175 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:55,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:55,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:55,178 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:55,178 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:55,187 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:55,187 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:55,191 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:55,191 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:55,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:55,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:55,194 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:55,194 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:55,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:55,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:55,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:55,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:55,203 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:55,203 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:55,204 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:55,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:56,903 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:55:58,321 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,321 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,325 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,325 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,326 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,327 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,328 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,329 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,335 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,335 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,337 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,337 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,525 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,530 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,530 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,532 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,533 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,534 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,535 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,535 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,536 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,646 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,649 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,653 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,661 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,873 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,873 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,877 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,878 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,878 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,879 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,880 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,881 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,885 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,886 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,887 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,888 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,890 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,890 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,891 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,971 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,972 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:55:58,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:00,312 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:00,312 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:00,315 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:00,316 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:00,316 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:00,317 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:00,318 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:00,319 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:00,323 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:00,323 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:00,325 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:00,325 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:00,327 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:00,327 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:00,328 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:00,328 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:00,905 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:56:01,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:01,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:01,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:01,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:01,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:01,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:01,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:01,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:01,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:01,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:01,683 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:01,683 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:01,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:01,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:01,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:01,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:01,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:01,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:01,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:01,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:01,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:01,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:01,793 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:01,793 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:01,797 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:01,797 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:01,799 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:01,799 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:01,801 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:01,801 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:01,802 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:01,802 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:01,995 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:01,995 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:01,999 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:01,999 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:02,000 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:02,000 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:02,001 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:02,002 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:02,006 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:02,006 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:02,008 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:02,008 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:02,010 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:02,010 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:02,011 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:02,012 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:03,588 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:03,589 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:03,592 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:03,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:03,594 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:03,594 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:03,596 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:03,596 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:03,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:03,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:03,602 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:03,602 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:03,604 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:03,604 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:03,605 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:03,605 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:04,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 18:56:04,791 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 18:56:04,906 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:56:06,907 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:56:07,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:07,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:07,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:07,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:07,646 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:07,646 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:07,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:07,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:07,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:07,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:07,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:07,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:07,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:07,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:07,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:07,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:09,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:09,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:09,439 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:09,439 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:09,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:09,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:09,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:09,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:09,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:09,447 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:09,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:09,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:09,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:09,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:09,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:09,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:10,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:10,866 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:10,869 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:10,870 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:10,870 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:10,871 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:10,872 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:10,873 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:10,877 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:10,877 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:10,878 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:10,879 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:10,880 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:10,881 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:10,881 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:10,882 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:10,909 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:56:12,518 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 18:56:12,910 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:56:14,911 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:56:16,245 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:16,246 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:16,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:16,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:16,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:16,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:16,251 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:16,252 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:16,255 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:16,256 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:16,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:16,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:16,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:16,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:16,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:16,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:18,913 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:56:20,013 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 18:56:20,013 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 18:56:20,914 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:56:21,586 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:21,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:21,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:21,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:21,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:21,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:21,592 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:21,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:21,596 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:21,597 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:21,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:21,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:21,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:21,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:21,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:21,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:21,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:21,675 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:21,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:21,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:21,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:21,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:21,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:21,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:21,684 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:21,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:21,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:21,687 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:21,688 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:21,689 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:21,689 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:21,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:22,914 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:56:26,916 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:56:28,917 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:56:31,918 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:56:33,919 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:56:34,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:34,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:34,736 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:34,737 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:34,737 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:34,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:34,739 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:34,739 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:34,742 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:34,743 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:34,744 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:34,744 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:34,746 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:34,746 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:34,747 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:34,747 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:35,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 18:56:35,360 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 18:56:35,920 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:56:37,921 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:56:40,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:40,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:40,602 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:40,603 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:40,603 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:40,604 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:40,605 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:40,606 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:40,608 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:40,609 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:40,610 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:40,610 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:40,612 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:40,612 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:40,613 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:40,613 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:56:41,922 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:56:42,588 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 18:56:43,923 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:56:45,924 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:56:47,925 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:56:49,926 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:56:50,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 18:56:50,583 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 18:56:51,927 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:56:53,928 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:56:55,929 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:56:57,930 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:56:59,931 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:57:01,932 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:57:03,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:03,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:03,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:03,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:03,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:03,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:03,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:03,513 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:03,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:03,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:03,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:03,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:03,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:03,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:03,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:03,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:03,933 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:57:05,812 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 18:57:05,812 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 18:57:05,934 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:57:07,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:07,344 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:07,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:07,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:07,348 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:07,348 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:07,349 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:07,350 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:07,351 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:07,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:07,353 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:07,353 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:07,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:07,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:07,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:07,356 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:07,935 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:57:08,033 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:08,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:08,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:08,037 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:08,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:08,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:08,039 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:08,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:08,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:08,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:08,043 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:08,043 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:08,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:08,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:08,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:08,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:09,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:09,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:09,127 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:09,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:09,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:09,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:09,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:09,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:09,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:09,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:09,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:09,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:09,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:09,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:09,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:09,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:09,936 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:57:10,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:10,433 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:10,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:10,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:10,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:10,437 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:10,438 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:10,438 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:10,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:10,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:10,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:10,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:10,443 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:10,444 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:10,444 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:10,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:11,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:11,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:11,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:11,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:11,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:11,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:11,577 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:11,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:11,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:11,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:11,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:11,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:11,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:11,583 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:11,583 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:11,584 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:11,649 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:11,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:11,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:11,653 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:11,653 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:11,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:11,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:11,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:11,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:11,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:11,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:11,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:11,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:11,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:11,661 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:11,661 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:11,936 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:57:12,661 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 18:57:13,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:13,101 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:13,103 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:13,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:13,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:13,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:13,106 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:13,106 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:13,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:13,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:13,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:13,110 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:13,111 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:13,111 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:13,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:13,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:13,937 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:57:15,248 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:15,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:15,251 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:15,251 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:15,252 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:15,252 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:15,253 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:15,254 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:15,255 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:15,256 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:15,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:15,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:15,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:15,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:15,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:15,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:15,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:15,766 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:15,768 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:15,769 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:15,770 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:15,771 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:15,773 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:15,774 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:15,775 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:15,776 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:15,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:15,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:15,779 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:15,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:15,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:15,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:15,938 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:57:16,919 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:16,920 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:16,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:16,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:16,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:16,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:16,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:16,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:16,929 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:16,929 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:16,929 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:16,930 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:16,930 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:16,931 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:16,931 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:16,932 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:17,939 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:57:18,992 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:18,993 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:18,994 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:18,994 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:19,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:19,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:19,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:19,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:19,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:19,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:19,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:19,048 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:19,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:19,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:19,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:19,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:19,052 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:19,052 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:19,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:19,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:19,170 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:19,176 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:19,176 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:19,176 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:19,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:19,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:19,178 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:19,178 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:19,940 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:57:21,033 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 18:57:21,033 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 18:57:21,127 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:21,127 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:21,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:21,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:21,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:21,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:21,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:21,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:21,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:21,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:21,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:21,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:21,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:21,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:21,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:21,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:21,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:21,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:21,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:21,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:21,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:21,427 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:21,429 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:21,429 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:21,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:21,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:21,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:21,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:21,444 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:21,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:21,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:21,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:21,619 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:21,620 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:21,624 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:21,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:21,626 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:21,626 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:21,628 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:21,628 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:21,639 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:21,639 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:21,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:21,642 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:21,644 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:21,644 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:21,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:21,646 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:21,941 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:57:23,491 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:23,491 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:23,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:23,496 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:23,496 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:23,497 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:23,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:23,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:23,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:23,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:23,508 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:23,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:23,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:23,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:23,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:23,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:25,944 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:57:26,218 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:26,218 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:26,222 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:26,223 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:26,224 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:26,224 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:26,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:26,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:26,233 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:26,233 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:26,235 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:26,235 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:26,237 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:26,237 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:26,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:26,239 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:26,291 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:26,291 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:26,295 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:26,295 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:26,296 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:26,297 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:26,298 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:26,299 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:26,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:26,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:26,307 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:26,308 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:26,309 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:26,310 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:26,311 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:26,311 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:26,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:26,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:26,388 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:26,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:26,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:26,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:26,392 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:26,392 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:26,398 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:26,399 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:26,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:26,401 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:26,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:26,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:26,404 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:26,404 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:29,946 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:57:30,684 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:30,684 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:30,689 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:30,689 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:30,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:30,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:30,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:30,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:30,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:30,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:30,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:30,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:30,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:30,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:30,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:30,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:33,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:33,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:33,716 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:33,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:33,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:33,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:33,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:33,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:33,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:33,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:33,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:33,725 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:33,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:33,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:33,728 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:33,728 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:33,948 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:57:34,334 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:34,334 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:34,339 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:34,339 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:34,340 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:34,340 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:34,342 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:34,342 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:34,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:34,348 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:34,350 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:34,350 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:34,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:34,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:34,353 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:34,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:36,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 18:57:36,258 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 18:57:37,805 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:37,806 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:37,810 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:37,810 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:37,811 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:37,811 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:37,813 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:37,813 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:37,818 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:37,819 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:37,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:37,821 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:37,822 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:37,823 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:37,823 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:37,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:37,949 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:57:40,951 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:40,951 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:40,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:40,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:40,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:40,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:40,958 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:40,958 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:40,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:40,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:40,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:40,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:40,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:40,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:40,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:40,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:41,951 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:57:42,737 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 18:57:42,799 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:42,800 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:42,804 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:42,804 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:42,805 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:42,805 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:42,807 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:42,807 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:42,812 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:42,812 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:42,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:42,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:42,816 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:42,816 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:42,817 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:42,817 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:43,952 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:57:44,410 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:44,410 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:44,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:44,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:44,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:44,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:44,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:44,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:44,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:44,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:44,424 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:44,424 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:44,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:44,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:44,427 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:44,428 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:44,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:44,437 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:44,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:44,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:44,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:44,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:44,443 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:44,444 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:44,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:44,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:44,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:44,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:44,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:44,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:44,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:44,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:47,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:47,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:47,267 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:47,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:47,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:47,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:47,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:47,272 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:47,276 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:47,276 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:47,278 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:47,279 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:47,281 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:47,281 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:47,282 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:47,283 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:47,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:47,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:47,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:47,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:47,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:47,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:47,688 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:47,688 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:47,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:47,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:47,694 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:47,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:47,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:47,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:47,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:47,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:47,954 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:57:48,740 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:48,740 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:48,744 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:48,744 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:48,745 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:48,746 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:48,747 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:48,747 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:48,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:48,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:48,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:48,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:48,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:48,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:48,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:48,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:50,194 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:50,194 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:50,198 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:50,198 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:50,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:50,200 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:50,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:50,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:50,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:50,206 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:50,207 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:50,208 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:50,209 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:50,210 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:50,210 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:50,211 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:50,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:50,687 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:50,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:50,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:50,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:50,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:50,694 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:50,694 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:50,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:50,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:50,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:50,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:50,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:50,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:50,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:50,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:50,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:50,792 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:50,796 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:50,796 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:50,797 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:50,797 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:50,799 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:50,799 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:50,803 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:50,803 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:50,805 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:50,805 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:50,807 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:50,807 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:50,808 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:50,809 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:51,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 18:57:51,660 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 18:57:51,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:51,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:51,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:51,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:51,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:51,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:51,783 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:51,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:51,787 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:51,788 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:51,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:51,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:51,792 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:51,792 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:51,793 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:51,793 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:51,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:51,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:51,844 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:51,845 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:51,845 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:51,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:51,847 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:51,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:51,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:51,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:51,853 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:51,854 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:51,855 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:51,856 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:51,856 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:51,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:51,918 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:51,918 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:51,921 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:51,922 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:51,922 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:51,923 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:51,924 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:51,925 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:51,929 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:51,929 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:51,931 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:51,931 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:51,933 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:51,933 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:51,934 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:51,934 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:51,955 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:57:53,628 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:53,628 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:53,632 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:53,632 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:53,633 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:53,634 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:53,635 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:53,635 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:53,639 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:53,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:53,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:53,642 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:53,643 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:53,644 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:53,644 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:53,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:53,956 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:57:55,006 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:55,007 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:55,010 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:55,011 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:55,012 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:55,012 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:55,013 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:55,014 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:55,018 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:55,018 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:55,020 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:55,020 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:55,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:55,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:55,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:55,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:56,275 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:56,276 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:56,279 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:56,279 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:56,280 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:56,280 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:56,282 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:56,282 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:56,286 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:56,287 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:56,288 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:56,288 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:56,290 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:56,290 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:56,291 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:56,292 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:57,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:57,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:57,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:57,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:57,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:57,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:57,664 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:57,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:57,669 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:57,669 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:57,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:57,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:57,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:57,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:57,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:57,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:57,958 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:57:59,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:59,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:59,412 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:59,413 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:59,413 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:59,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:59,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:59,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:59,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:59,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:59,421 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:59,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:59,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:59,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:59,424 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:59,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:57:59,959 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:58:00,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:00,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:00,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:00,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:00,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:00,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:00,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:00,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:00,585 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:00,585 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:00,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:00,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:00,589 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:00,589 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:00,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:00,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:01,960 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:58:03,466 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:03,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:03,470 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:03,470 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:03,471 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:03,472 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:03,473 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:03,473 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:03,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:03,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:03,479 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:03,479 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:03,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:03,481 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:03,482 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:03,482 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:05,961 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:58:06,904 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 18:58:06,904 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 18:58:07,962 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:58:08,464 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:08,465 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:08,468 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:08,468 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:08,469 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:08,470 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:08,471 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:08,471 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:08,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:08,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:08,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:08,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:08,479 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:08,479 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:08,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:08,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:09,963 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:58:12,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:12,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:12,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:12,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:12,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:12,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:12,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:12,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:12,577 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:12,577 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:12,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:12,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:12,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:12,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:12,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:12,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:12,814 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 18:58:13,965 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:58:15,965 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:58:17,195 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:17,196 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:17,198 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:17,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:17,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:17,200 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:17,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:17,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:17,204 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:17,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:17,206 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:17,207 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:17,208 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:17,209 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:17,209 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:17,210 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:17,966 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:58:19,967 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:58:21,968 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:58:22,189 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 18:58:22,190 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 18:58:23,969 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:58:25,970 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:58:26,855 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:26,856 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:26,858 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:26,859 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:26,859 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:26,860 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:26,861 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:26,861 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:26,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:26,864 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:26,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:26,866 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:26,867 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:26,867 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:26,868 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:26,868 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:27,971 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:58:29,972 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:58:30,317 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:30,317 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:30,320 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:30,321 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:30,322 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:30,322 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:30,323 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:30,324 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:30,326 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:30,327 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:30,328 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:30,329 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:30,330 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:30,330 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:30,331 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:30,332 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:31,972 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:58:33,973 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:58:35,974 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:58:37,411 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 18:58:37,411 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 18:58:37,975 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:58:39,976 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:58:41,977 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:58:42,890 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 18:58:43,978 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:58:45,979 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:58:47,980 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:58:49,981 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:58:51,982 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:58:52,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 18:58:52,658 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 18:58:53,813 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:53,813 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:53,816 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:53,817 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:53,817 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:53,817 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:53,818 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:53,819 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:53,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:53,821 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:53,822 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:53,822 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:53,823 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:53,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:53,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:53,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:53,983 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:58:55,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:55,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:55,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:55,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:55,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:55,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:55,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:55,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:55,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:55,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:55,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:55,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:55,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:55,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:55,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:55,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:56,984 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:58:57,275 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:57,275 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:57,278 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:57,278 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:57,279 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:57,279 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:57,280 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:57,281 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:57,282 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:57,283 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:57,284 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:57,284 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:57,285 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:57,286 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:57,286 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:57,287 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:58,633 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:58,634 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:58,637 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:58,637 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:58,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:58,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:58,639 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:58,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:58,642 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:58,642 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:58,643 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:58,644 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:58,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:58,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:58,646 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:58,647 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:58,985 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:58:59,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:59,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:59,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:59,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:59,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:59,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:59,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:59,048 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:59,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:59,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:59,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:59,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:59,052 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:59,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:59,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:59,054 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:59,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:59,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:59,732 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:59,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:59,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:59,734 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:59,735 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:59,735 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:59,737 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:59,737 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:59,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:59,739 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:59,740 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:59,740 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:59,741 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:58:59,742 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:00,186 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:00,186 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:00,189 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:00,190 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:00,190 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:00,191 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:00,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:00,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:00,193 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:00,194 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:00,195 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:00,196 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:00,197 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:00,197 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:00,198 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:00,198 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:00,986 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:59:02,144 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:02,144 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:02,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:02,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:02,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:02,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:02,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:02,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:02,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:02,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:02,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:02,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:02,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:02,155 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:02,155 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:02,156 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:02,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:02,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:02,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:02,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:02,707 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:02,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:02,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:02,710 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:02,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:02,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:02,713 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:02,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:02,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:02,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:02,716 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:02,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:02,987 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:59:04,165 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:04,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:04,168 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:04,168 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:04,169 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:04,169 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:04,170 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:04,171 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:04,172 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:04,173 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:04,174 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:04,174 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:04,175 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:04,176 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:04,176 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:04,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:04,988 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:59:05,029 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:05,030 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:05,030 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:05,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:05,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:05,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:05,169 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:05,169 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:05,170 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:05,170 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:05,171 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:05,172 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:05,173 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:05,173 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:05,174 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:05,175 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:05,176 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:05,176 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:05,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:05,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:05,204 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:05,208 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:05,209 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:05,210 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:05,210 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:05,211 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:05,211 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:05,212 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:05,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:05,469 DEBUG SenderThread:3760704 [sender.py:send():232] send: history +2022-08-30 18:59:05,684 DEBUG SenderThread:3760704 [sender.py:send():232] send: summary +2022-08-30 18:59:05,759 INFO SenderThread:3760704 [sender.py:_save_file():946] saving file wandb-summary.json with policy end +2022-08-30 18:59:05,989 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/wandb-summary.json +2022-08-30 18:59:06,989 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:59:07,990 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:59:08,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 18:59:08,160 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 18:59:12,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:12,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:12,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:12,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:12,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:12,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:12,229 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:12,229 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:12,235 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:12,236 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:12,237 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:12,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:12,240 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:12,240 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:12,241 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:12,241 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:12,988 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 18:59:13,007 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:59:16,218 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:16,219 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:16,223 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:16,223 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:16,224 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:16,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:16,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:16,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:16,233 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:16,233 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:16,235 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:16,235 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:16,237 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:16,237 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:16,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:16,239 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:17,009 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:59:18,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:18,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:18,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:18,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:18,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:18,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:18,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:18,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:18,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:18,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:18,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:18,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:18,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:18,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:18,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:18,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:21,010 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:59:21,776 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:21,776 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:21,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:21,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:21,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:21,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:21,783 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:21,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:21,788 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:21,789 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:21,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:21,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:21,793 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:21,793 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:21,794 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:21,794 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:21,970 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:21,970 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:21,974 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:21,974 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:21,975 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:21,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:21,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:21,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:21,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:21,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:21,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:21,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:21,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:21,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:21,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:21,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:22,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:22,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:22,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:22,039 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:22,039 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:22,039 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:22,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:22,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:22,043 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:22,044 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:22,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:22,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:22,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:22,048 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:22,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:22,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:22,075 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:22,075 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:22,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:22,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:22,080 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:22,080 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:22,082 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:22,082 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:22,087 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:22,088 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:22,090 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:22,090 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:22,092 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:22,092 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:22,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:22,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:23,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 18:59:23,566 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 18:59:23,583 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:23,583 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:23,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:23,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:23,588 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:23,588 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:23,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:23,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:23,595 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:23,596 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:23,597 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:23,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:23,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:23,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:23,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:23,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:25,012 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:59:27,078 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:27,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:27,083 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:27,083 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:27,084 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:27,084 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:27,086 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:27,086 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:27,090 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:27,091 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:27,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:27,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:27,095 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:27,095 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:27,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:27,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:29,014 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:59:29,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:29,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:29,206 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:29,207 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:29,207 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:29,208 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:29,210 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:29,210 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:29,215 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:29,215 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:29,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:29,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:29,219 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:29,219 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:29,220 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:29,220 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:29,349 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:29,349 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:29,353 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:29,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:29,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:29,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:29,357 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:29,357 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:29,361 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:29,362 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:29,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:29,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:29,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:29,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:29,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:29,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:31,015 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:59:31,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:31,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:31,039 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:31,039 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:31,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:31,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:31,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:31,043 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:31,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:31,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:31,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:31,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:31,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:31,052 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:31,052 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:31,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:32,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:32,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:32,595 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:32,595 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:32,596 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:32,596 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:32,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:32,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:32,602 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:32,603 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:32,605 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:32,605 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:32,607 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:32,607 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:32,608 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:32,608 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:35,018 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:59:35,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:35,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:35,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:35,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:35,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:35,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:35,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:35,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:35,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:35,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:35,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:35,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:35,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:35,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:35,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:35,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:35,537 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:35,538 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:35,541 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:35,541 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:35,542 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:35,543 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:35,544 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:35,545 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:35,548 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:35,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:35,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:35,551 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:35,552 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:35,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:35,554 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:35,554 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:38,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:38,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:38,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:38,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:38,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:38,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:38,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:38,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:38,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:38,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:38,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:38,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:38,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:38,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:38,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:38,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:38,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:38,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:38,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:38,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:38,734 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:38,735 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:38,736 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:38,736 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:38,740 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:38,740 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:38,742 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:38,742 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:38,744 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:38,744 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:38,745 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:38,746 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:38,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 18:59:38,791 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 18:59:39,020 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:59:40,397 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:40,398 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:40,402 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:40,402 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:40,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:40,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:40,405 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:40,405 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:40,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:40,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:40,411 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:40,411 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:40,413 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:40,413 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:40,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:40,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:41,021 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:59:41,234 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:41,234 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:41,237 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:41,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:41,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:41,239 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:41,240 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:41,241 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:41,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:41,245 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:41,246 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:41,247 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:41,248 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:41,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:41,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:41,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:41,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:41,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:41,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:41,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:41,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:41,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:41,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:41,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:41,762 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:41,762 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:41,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:41,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:41,766 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:41,766 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:41,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:41,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:42,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:42,964 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:42,971 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:42,971 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:42,971 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:42,971 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:42,971 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:42,972 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:42,974 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:42,975 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:42,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:42,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:42,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:42,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:42,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:42,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:43,065 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 18:59:44,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,416 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,421 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,427 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,428 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,429 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,431 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,433 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,485 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,485 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,486 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,487 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,491 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,491 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,496 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,497 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,707 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:44,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:45,022 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:59:45,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:45,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:45,868 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:45,869 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:45,869 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:45,870 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:45,871 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:45,872 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:45,875 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:45,876 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:45,877 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:45,878 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:45,879 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:45,880 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:45,881 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:45,881 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:47,024 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:59:49,025 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:59:51,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:51,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:51,832 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:51,832 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:51,833 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:51,833 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:51,834 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:51,835 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:51,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:51,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:51,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:51,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:51,842 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:51,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:51,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:51,844 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:51,885 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:51,886 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:51,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:51,890 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:51,890 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:51,891 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:51,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:51,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:51,896 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:51,896 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:51,898 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:51,898 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:51,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:51,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:51,901 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:51,901 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:52,888 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:52,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:52,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:52,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:52,893 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:52,893 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:52,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:52,895 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:52,898 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:52,898 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:52,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:52,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:52,902 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:52,902 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:52,903 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:52,903 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:53,027 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:59:54,010 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 18:59:54,010 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 18:59:54,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:54,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:54,346 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:54,346 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:54,346 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:54,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:54,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:54,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:54,348 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:54,348 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:54,350 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:54,350 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:54,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:54,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:54,353 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:54,353 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:55,028 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:59:55,642 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:55,642 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:55,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:55,646 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:55,646 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:55,647 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:55,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:55,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:55,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:55,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:55,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:55,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:55,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:55,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:55,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:55,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:57,029 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 18:59:59,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:59,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:59,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:59,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:59,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:59,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:59,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:59,513 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:59,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:59,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:59,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:59,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:59,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:59,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:59,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 18:59:59,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:01,031 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:00:01,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:01,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:01,854 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:01,854 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:01,855 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:01,856 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:01,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:01,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:01,860 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:01,861 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:01,862 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:01,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:01,864 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:01,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:01,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:01,866 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:03,031 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:00:03,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:03,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:03,167 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:03,168 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:03,168 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:03,169 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:03,170 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:03,170 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:03,173 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:03,174 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:03,175 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:03,176 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:03,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:03,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:03,178 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:03,179 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:05,032 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:00:06,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:06,783 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:06,785 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:06,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:06,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:06,787 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:06,788 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:06,789 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:06,792 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:06,792 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:06,793 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:06,794 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:06,795 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:06,796 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:06,796 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:06,797 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:07,033 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:00:09,034 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:00:09,240 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:00:09,240 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:00:13,036 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:00:13,139 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:00:15,037 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:00:15,431 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:15,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:15,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:15,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:15,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:15,437 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:15,438 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:15,438 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:15,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:15,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:15,443 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:15,443 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:15,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:15,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:15,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:15,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:17,037 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:00:19,038 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:00:21,039 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:00:22,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:22,687 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:22,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:22,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:22,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:22,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:22,693 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:22,693 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:22,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:22,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:22,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:22,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:22,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:22,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:22,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:22,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:23,040 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:00:24,539 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:00:24,540 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:00:25,041 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:00:27,041 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:00:29,042 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:00:31,043 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:00:33,044 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:00:35,045 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:00:37,046 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:00:39,046 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:00:39,763 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:00:39,763 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:00:41,047 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:00:43,048 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:00:43,209 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:00:45,049 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:00:46,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:46,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:46,676 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:46,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:46,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:46,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:46,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:46,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:46,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:46,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:46,682 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:46,683 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:46,684 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:46,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:46,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:46,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:47,050 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:00:47,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:47,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:47,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:47,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:47,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:47,375 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:47,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:47,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:47,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:47,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:47,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:47,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:47,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:47,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:47,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:47,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:47,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:47,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:47,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:47,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:47,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:47,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:47,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:47,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:47,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:47,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:47,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:47,710 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:47,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:47,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:47,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:47,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:48,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:48,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:48,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:48,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:48,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:48,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:48,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:48,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:48,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:48,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:48,388 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:48,388 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:48,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:48,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:48,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:48,391 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:48,918 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:48,918 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:48,921 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:48,921 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:48,922 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:48,923 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:48,923 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:48,924 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:48,925 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:48,926 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:48,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:48,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:48,929 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:48,929 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:48,930 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:48,930 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:49,051 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:00:50,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:50,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:50,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:50,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:50,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:50,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:50,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:50,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:50,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:50,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:50,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:50,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:50,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:50,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:50,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:50,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:50,964 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:50,964 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:50,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:50,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:50,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:50,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:50,969 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:50,969 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:50,971 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:50,971 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:50,972 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:50,973 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:50,974 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:50,974 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:50,975 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:50,975 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:51,051 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:00:52,826 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:52,826 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:52,829 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:52,829 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:52,830 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:52,830 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:52,831 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:52,832 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:52,833 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:52,833 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:52,834 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:52,835 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:52,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:52,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:52,837 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:52,837 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:53,052 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:00:53,393 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:53,394 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:53,396 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:53,396 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:53,397 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:53,397 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:53,398 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:53,399 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:53,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:53,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:53,401 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:53,402 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:53,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:53,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:53,404 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:53,404 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:53,672 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:53,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:53,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:53,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:53,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:53,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:53,682 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:53,683 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:53,684 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:53,684 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:53,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:53,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:53,688 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:53,688 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:53,689 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:53,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:53,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:53,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:53,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:53,693 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:53,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:53,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:53,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:53,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:53,853 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:53,854 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:53,854 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:53,854 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:00:55,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:00:55,036 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:00:55,053 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:01:01,055 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:01:03,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:03,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:03,375 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:03,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:03,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:03,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:03,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:03,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:03,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:03,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:03,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:03,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:03,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:03,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:03,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:03,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:05,057 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:01:06,921 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:06,921 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:06,925 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:06,925 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:06,926 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:06,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:06,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:06,929 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:06,934 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:06,935 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:06,936 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:06,937 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:06,938 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:06,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:06,940 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:06,940 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:09,059 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:01:10,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:01:10,261 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:01:10,948 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:10,949 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:10,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:10,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:10,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:10,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:10,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:10,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:10,962 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:10,962 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:10,964 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:10,964 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:10,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:10,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:10,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:10,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:12,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:12,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:12,689 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:12,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:12,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:12,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:12,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:12,693 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:12,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:12,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:12,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:12,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:12,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:12,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:12,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:12,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:13,061 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:01:13,289 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:01:14,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:14,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:14,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:14,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:14,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:14,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:14,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:14,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:14,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:14,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:14,589 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:14,589 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:14,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:14,592 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:14,592 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:14,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:17,063 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:01:18,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:18,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:18,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:18,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:18,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:18,106 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:18,107 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:18,107 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:18,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:18,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:18,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:18,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:18,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:18,117 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:18,118 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:18,118 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:18,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:18,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:18,388 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:18,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:18,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:18,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:18,391 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:18,392 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:18,397 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:18,397 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:18,399 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:18,399 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:18,401 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:18,401 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:18,402 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:18,402 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:19,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:19,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:19,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:19,649 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:19,649 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:19,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:19,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:19,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:19,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:19,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:19,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:19,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:19,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:19,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:19,661 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:19,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:21,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:21,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:21,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:21,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:21,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:21,127 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:21,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:21,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:21,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:21,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:21,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:21,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:21,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:21,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:21,138 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:21,138 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:21,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:21,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:21,253 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:21,254 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:21,254 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:21,255 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:21,256 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:21,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:21,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:21,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:21,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:21,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:21,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:21,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:21,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:21,267 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:22,065 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:01:23,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:23,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:23,471 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:23,472 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:23,473 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:23,473 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:23,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:23,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:23,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:23,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:23,482 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:23,482 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:23,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:23,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:23,485 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:23,486 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:24,066 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:01:25,551 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:01:25,552 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:01:26,299 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:26,300 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:26,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:26,304 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:26,304 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:26,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:26,306 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:26,307 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:26,311 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:26,311 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:26,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:26,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:26,315 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:26,315 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:26,316 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:26,316 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:26,486 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:26,486 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:26,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:26,491 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:26,491 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:26,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:26,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:26,494 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:26,497 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:26,498 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:26,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:26,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:26,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:26,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:26,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:26,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:28,067 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:01:30,068 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:01:30,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:30,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:30,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:30,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:30,762 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:30,762 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:30,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:30,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:30,768 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:30,769 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:30,770 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:30,771 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:30,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:30,773 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:30,773 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:30,774 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:32,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:32,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:32,029 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:32,029 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:32,030 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:32,030 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:32,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:32,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:32,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:32,037 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:32,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:32,039 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:32,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:32,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:32,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:32,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:32,462 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:32,463 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:32,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:32,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:32,468 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:32,468 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:32,470 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:32,470 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:32,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:32,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:32,476 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:32,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:32,478 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:32,479 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:32,479 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:32,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:33,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:33,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:33,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:33,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:33,424 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:33,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:33,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:33,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:33,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:33,431 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:33,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:33,433 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:33,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:33,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:33,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:33,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:33,545 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:33,546 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:33,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:33,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:33,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:33,551 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:33,552 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:33,552 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:33,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:33,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:33,558 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:33,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:33,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:33,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:33,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:33,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:34,069 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:01:35,300 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:35,301 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:35,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:35,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:35,306 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:35,306 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:35,308 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:35,308 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:35,312 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:35,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:35,314 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:35,315 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:35,316 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:35,316 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:35,317 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:35,318 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:36,070 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:01:36,543 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:36,544 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:36,547 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:36,548 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:36,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:36,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:36,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:36,551 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:36,555 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:36,555 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:36,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:36,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:36,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:36,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:36,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:36,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:38,071 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:01:39,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:39,245 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:39,248 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:39,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:39,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:39,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:39,251 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:39,251 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:39,255 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:39,255 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:39,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:39,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:39,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:39,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:39,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:39,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:40,359 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:40,359 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:40,362 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:40,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:40,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:40,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:40,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:40,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:40,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:40,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:40,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:40,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:40,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:40,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:40,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:40,375 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:40,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:40,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:40,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:40,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:40,431 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:40,431 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:40,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:40,433 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:40,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:40,437 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:40,438 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:40,439 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:40,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:40,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:40,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:40,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:40,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:01:40,777 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:01:41,899 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:41,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:41,904 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:41,904 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:41,905 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:41,905 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:41,906 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:41,907 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:41,910 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:41,911 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:41,912 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:41,912 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:41,914 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:41,914 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:41,915 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:41,915 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:42,072 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:01:42,804 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:42,804 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:42,807 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:42,808 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:42,809 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:42,809 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:42,810 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:42,811 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:42,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:42,815 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:42,816 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:42,816 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:42,818 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:42,818 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:42,819 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:42,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:42,882 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:42,882 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:42,885 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:42,886 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:42,887 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:42,887 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:42,888 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:42,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:42,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:42,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:42,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:42,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:42,896 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:42,896 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:42,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:42,898 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:43,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:43,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:43,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:43,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:43,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:43,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:43,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:43,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:43,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:43,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:43,074 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:43,074 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:43,076 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:43,076 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:43,077 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:43,077 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:43,362 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:01:44,073 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:01:46,074 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:01:46,664 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:46,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:46,668 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:46,668 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:46,669 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:46,669 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:46,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:46,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:46,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:46,675 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:46,676 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:46,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:46,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:46,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:46,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:46,680 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:46,797 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:46,798 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:46,801 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:46,801 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:46,802 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:46,802 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:46,803 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:46,804 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:46,807 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:46,808 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:46,809 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:46,809 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:46,811 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:46,811 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:46,812 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:46,813 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:48,186 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:48,186 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:48,190 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:48,190 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:48,191 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:48,191 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:48,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:48,193 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:48,196 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:48,196 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:48,198 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:48,198 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:48,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:48,200 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:48,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:48,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:49,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:49,039 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:49,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:49,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:49,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:49,043 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:49,044 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:49,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:49,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:49,048 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:49,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:49,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:49,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:49,052 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:49,052 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:49,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:49,275 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:49,276 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:49,279 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:49,279 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:49,280 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:49,280 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:49,282 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:49,282 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:49,285 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:49,286 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:49,287 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:49,288 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:49,289 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:49,289 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:49,290 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:49,291 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:50,076 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:01:52,077 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:01:54,078 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:01:55,969 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:55,969 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:55,972 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:55,973 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:55,973 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:55,974 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:55,975 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:55,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:55,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:55,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:55,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:55,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:55,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:55,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:55,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:55,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:56,079 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:01:56,082 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:01:56,082 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:01:56,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:56,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:56,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:56,144 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:56,144 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:56,145 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:56,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:56,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:56,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:56,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:56,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:56,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:56,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:56,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:56,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:56,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:57,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:57,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:57,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:57,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:57,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:57,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:57,391 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:57,392 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:57,395 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:57,395 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:57,396 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:57,397 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:57,398 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:57,399 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:57,399 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:57,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:01:58,080 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:02:00,082 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:02:00,546 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:00,547 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:00,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:00,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:00,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:00,551 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:00,552 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:00,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:00,555 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:00,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:00,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:00,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:00,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:00,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:00,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:00,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:02,082 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:02:05,805 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:05,806 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:05,808 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:05,809 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:05,809 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:05,810 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:05,811 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:05,811 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:05,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:05,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:05,815 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:05,816 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:05,817 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:05,817 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:05,818 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:05,818 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:06,084 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:02:08,085 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:02:10,086 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:02:11,368 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:02:11,368 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:02:12,087 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:02:12,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:12,940 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:12,943 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:12,943 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:12,944 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:12,944 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:12,945 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:12,946 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:12,948 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:12,949 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:12,950 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:12,950 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:12,951 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:12,952 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:12,952 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:12,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:13,431 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:02:14,088 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:02:16,089 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:02:18,090 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:02:20,091 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:02:22,092 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:02:24,092 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:02:26,093 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:02:26,615 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:02:26,616 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:02:28,094 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:02:30,094 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:02:32,095 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:02:34,096 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:02:36,097 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:02:36,496 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:36,496 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:36,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:36,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:36,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:36,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:36,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:36,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:36,504 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:36,504 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:36,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:36,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:36,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:36,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:36,508 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:36,508 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:37,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:37,421 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:37,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:37,424 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:37,424 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:37,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:37,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:37,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:37,427 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:37,428 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:37,429 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:37,429 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:37,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:37,431 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:37,431 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:37,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:37,993 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:37,993 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:37,996 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:37,996 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:37,997 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:37,997 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:37,998 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:37,999 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:38,000 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:38,000 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:38,001 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:38,002 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:38,003 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:38,003 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:38,004 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:38,004 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:38,098 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:02:38,496 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:38,497 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:38,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:38,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:38,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:38,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:38,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:38,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:38,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:38,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:38,504 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:38,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:38,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:38,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:38,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:38,508 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:39,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:39,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:39,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:39,602 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:39,602 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:39,603 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:39,604 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:39,604 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:39,605 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:39,606 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:39,607 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:39,607 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:39,608 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:39,609 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:39,609 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:39,610 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:40,098 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:02:41,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:41,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:41,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:41,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:41,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:41,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:41,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:41,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:41,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:41,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:41,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:41,388 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:41,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:41,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:41,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:41,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:41,610 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:41,611 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:41,613 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:41,614 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:41,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:41,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:41,716 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:41,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:41,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:41,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:41,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:41,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:41,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:41,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:41,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:41,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:41,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:41,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:41,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:41,725 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:41,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:41,787 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:41,787 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:41,788 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:41,789 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:41,789 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:41,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:41,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:41,844 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:02:41,844 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:02:41,964 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:42,119 DEBUG SenderThread:3760704 [sender.py:send():232] send: history +2022-08-30 19:02:42,154 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:02:42,191 DEBUG SenderThread:3760704 [sender.py:send():232] send: summary +2022-08-30 19:02:42,249 INFO SenderThread:3760704 [sender.py:_save_file():946] saving file wandb-summary.json with policy end +2022-08-30 19:02:43,144 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/wandb-summary.json +2022-08-30 19:02:43,553 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:02:44,144 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:02:46,289 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:46,290 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:46,293 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:46,293 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:46,294 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:46,295 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:46,296 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:46,297 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:46,304 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:46,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:46,307 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:46,307 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:46,309 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:46,309 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:46,310 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:46,311 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:46,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:46,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:46,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:46,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:46,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:46,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:46,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:46,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:46,586 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:46,586 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:46,588 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:46,589 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:46,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:46,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:46,592 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:46,592 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:48,146 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:02:54,149 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:02:56,150 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:02:57,315 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:02:57,315 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:02:58,541 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:58,542 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:58,546 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:58,547 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:58,548 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:58,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:58,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:58,551 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:58,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:58,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:58,558 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:58,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:58,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:58,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:58,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:02:58,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:01,152 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:03:04,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:04,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:04,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:04,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:04,262 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:04,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:04,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:04,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:04,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:04,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:04,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:04,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:04,273 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:04,273 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:04,274 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:04,275 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:05,153 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:03:07,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:07,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:07,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:07,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:07,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:07,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:07,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:07,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:07,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:07,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:07,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:07,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:07,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:07,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:07,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:07,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:07,811 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:07,811 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:07,815 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:07,816 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:07,816 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:07,817 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:07,818 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:07,819 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:07,823 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:07,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:07,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:07,826 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:07,827 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:07,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:07,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:07,829 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:08,949 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:08,950 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:08,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:08,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:08,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:08,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:08,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:08,957 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:08,961 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:08,961 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:08,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:08,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:08,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:08,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:08,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:08,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:09,155 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:03:10,514 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:10,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:10,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:10,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:10,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:10,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:10,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:10,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:10,525 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:10,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:10,527 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:10,527 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:10,529 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:10,529 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:10,530 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:10,531 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:10,834 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:10,835 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:10,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:10,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:10,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:10,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:10,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:10,842 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:10,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:10,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:10,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:10,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:10,850 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:10,850 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:10,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:10,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:10,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:10,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:10,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:10,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:10,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:10,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:10,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:10,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:10,992 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:10,993 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:10,994 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:10,995 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:10,996 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:10,997 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:10,997 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:10,998 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:11,155 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:03:12,544 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:03:12,544 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:03:13,628 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:03:14,091 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:14,092 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:14,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:14,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:14,097 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:14,097 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:14,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:14,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:14,103 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:14,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:14,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:14,106 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:14,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:14,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:14,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:14,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:15,094 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:15,094 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:15,098 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:15,098 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:15,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:15,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:15,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:15,101 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:15,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:15,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:15,106 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:15,107 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:15,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:15,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:15,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:15,110 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:15,157 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:03:15,197 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:15,198 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:15,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:15,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:15,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:15,203 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:15,204 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:15,204 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:15,208 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:15,209 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:15,210 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:15,211 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:15,212 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:15,213 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:15,213 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:15,214 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:16,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:16,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:16,457 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:16,458 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:16,458 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:16,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:16,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:16,461 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:16,465 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:16,465 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:16,466 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:16,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:16,468 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:16,469 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:16,470 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:16,470 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:17,157 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:03:17,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:17,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:17,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:17,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:17,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:17,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:17,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:17,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:17,992 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:17,992 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:17,994 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:17,994 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:17,996 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:17,996 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:17,997 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:17,997 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:19,919 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:19,920 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:19,924 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:19,924 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:19,925 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:19,926 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:19,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:19,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:19,932 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:19,932 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:19,934 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:19,934 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:19,936 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:19,936 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:19,937 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:19,938 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:21,159 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:03:21,314 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:21,314 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:21,318 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:21,319 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:21,319 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:21,320 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:21,321 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:21,321 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:21,325 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:21,326 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:21,327 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:21,328 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:21,329 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:21,330 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:21,330 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:21,331 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:22,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:22,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:22,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:22,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:22,424 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:22,424 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:22,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:22,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:22,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:22,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:22,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:22,433 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:22,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:22,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:22,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:22,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:23,159 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:03:24,021 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:24,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:24,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:24,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:24,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:24,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:24,028 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:24,029 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:24,033 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:24,033 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:24,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:24,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:24,037 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:24,037 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:24,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:24,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:25,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:25,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:25,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:25,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:25,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:25,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:25,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:25,267 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:25,270 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:25,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:25,272 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:25,273 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:25,274 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:25,274 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:25,275 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:25,276 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:27,161 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:03:27,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:03:27,842 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:03:29,162 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:03:29,212 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:29,213 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:29,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:29,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:29,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:29,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:29,219 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:29,219 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:29,223 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:29,223 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:29,224 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:29,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:29,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:29,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:29,228 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:29,228 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:29,439 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:29,439 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:29,443 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:29,443 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:29,444 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:29,444 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:29,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:29,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:29,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:29,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:29,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:29,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:29,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:29,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:29,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:29,455 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:31,162 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:03:31,890 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:31,890 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:31,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:31,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:31,895 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:31,895 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:31,896 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:31,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:31,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:31,901 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:31,902 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:31,903 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:31,904 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:31,904 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:31,905 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:31,906 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:33,012 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:33,013 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:33,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:33,017 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:33,017 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:33,018 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:33,019 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:33,020 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:33,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:33,024 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:33,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:33,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:33,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:33,028 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:33,028 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:33,029 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:33,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:33,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:33,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:33,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:33,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:33,141 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:33,142 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:33,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:33,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:33,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:33,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:33,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:33,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:33,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:33,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:33,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:35,164 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:03:35,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:35,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:35,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:35,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:35,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:35,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:35,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:35,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:35,375 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:35,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:35,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:35,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:35,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:35,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:35,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:35,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:37,165 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:03:37,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:37,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:37,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:37,759 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:37,759 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:37,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:37,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:37,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:37,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:37,765 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:37,766 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:37,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:37,768 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:37,769 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:37,769 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:37,770 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:37,879 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:37,879 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:37,882 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:37,883 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:37,883 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:37,884 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:37,885 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:37,885 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:37,888 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:37,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:37,890 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:37,891 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:37,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:37,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:37,893 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:37,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:37,972 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:37,972 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:37,975 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:37,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:37,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:37,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:37,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:37,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:37,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:37,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:37,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:37,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:37,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:37,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:37,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:37,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,054 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,059 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,765 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,766 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,769 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,770 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,771 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,771 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,773 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,776 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,776 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,847 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,847 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,854 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,854 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,859 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,859 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,861 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,861 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,862 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,948 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,949 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,952 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,952 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,958 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,959 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,960 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,961 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,962 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:38,964 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:39,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:39,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:39,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:39,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:39,155 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:39,155 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:39,157 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:39,157 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:39,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:39,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:39,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:39,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:39,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:39,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:39,165 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:39,165 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:39,166 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:03:40,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:40,339 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:40,342 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:40,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:40,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:40,344 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:40,345 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:40,345 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:40,348 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:40,349 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:40,350 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:40,351 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:40,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:40,353 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:40,353 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:40,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:41,167 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:03:43,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:03:43,116 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:03:43,169 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:03:43,697 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:03:45,170 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:03:47,171 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:03:49,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:49,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:49,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:49,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:49,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:49,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:49,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:49,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:49,117 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:49,118 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:49,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:49,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:49,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:49,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:49,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:49,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:50,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:50,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:50,234 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:50,234 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:50,235 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:50,235 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:50,235 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:50,236 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:50,236 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:50,236 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:50,237 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:50,237 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:50,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:50,239 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:50,239 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:50,240 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:51,172 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:03:51,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:51,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:51,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:51,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:51,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:51,427 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:51,428 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:51,428 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:51,431 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:51,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:51,433 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:51,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:51,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:51,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:51,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:51,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:53,173 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:03:54,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:54,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:54,358 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:54,358 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:54,359 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:54,359 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:54,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:54,361 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:54,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:54,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:54,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:54,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:54,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:54,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:54,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:54,368 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:55,174 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:03:57,175 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:03:58,341 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:03:58,341 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:03:59,176 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:03:59,931 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:59,931 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:59,934 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:59,934 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:59,935 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:59,936 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:59,937 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:59,937 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:59,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:59,940 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:59,941 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:59,941 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:59,942 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:59,943 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:59,944 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:03:59,944 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:01,177 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:04:02,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:02,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:02,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:02,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:02,117 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:02,118 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:02,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:02,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:02,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:02,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:02,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:02,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:02,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:02,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:02,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:02,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:03,178 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:04:05,179 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:04:07,180 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:04:09,181 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:04:11,182 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:04:13,183 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:04:13,675 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:04:13,676 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:04:13,919 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:04:15,184 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:04:17,185 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:04:19,186 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:04:21,187 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:04:23,188 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:04:25,189 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:04:26,155 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:26,155 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:26,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:26,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:26,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:26,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:26,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:26,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:26,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:26,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:26,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:26,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:26,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:26,165 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:26,165 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:26,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:26,284 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:26,284 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:26,287 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:26,287 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:26,288 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:26,288 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:26,289 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:26,290 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:26,291 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:26,292 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:26,293 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:26,293 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:26,294 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:26,295 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:26,295 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:26,296 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:27,190 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:04:28,255 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:28,256 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:28,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:28,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:28,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:28,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:28,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:28,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:28,262 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:28,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:28,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:28,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:28,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:28,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:28,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:28,267 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:28,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:28,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:28,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:28,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:28,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:28,583 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:28,584 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:28,584 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:28,586 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:28,586 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:28,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:28,588 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:28,589 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:28,589 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:28,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:28,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:28,947 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:04:28,948 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:04:29,191 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:04:29,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:29,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:29,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:29,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:29,548 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:29,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:29,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:29,554 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:29,555 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:29,555 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:29,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:29,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:31,192 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:04:32,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:32,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:32,075 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:32,076 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:32,076 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:32,077 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:32,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:32,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:32,088 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:32,089 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:32,091 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:32,091 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:32,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:32,094 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:32,094 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:32,095 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:36,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:36,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:36,765 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:36,766 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:36,766 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:36,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:36,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:36,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:36,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:36,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:36,774 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:36,775 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:36,776 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:36,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:36,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:36,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:37,194 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:04:38,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:38,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:38,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:38,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:38,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:38,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:38,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:38,375 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:38,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:38,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:38,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:38,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:38,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:38,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:38,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:38,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:39,195 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:04:39,991 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:39,992 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:39,995 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:39,996 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:39,997 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:39,997 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:39,999 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:39,999 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:40,004 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:40,005 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:40,007 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:40,007 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:40,009 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:40,009 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:40,010 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:40,011 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:40,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:40,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:40,206 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:40,206 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:40,207 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:40,207 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:40,209 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:40,209 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:40,215 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:40,215 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:40,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:40,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:40,219 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:40,220 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:40,220 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:40,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:43,197 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:04:43,848 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:04:44,170 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:04:44,170 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:04:48,200 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:04:52,202 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:04:52,280 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:52,281 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:52,285 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:52,286 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:52,287 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:52,288 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:52,289 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:52,290 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:52,294 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:52,295 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:52,296 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:52,297 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:52,299 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:52,299 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:52,300 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:52,301 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:56,203 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:04:58,205 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:04:58,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:58,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:58,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:58,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:58,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:58,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:58,827 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:58,827 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:58,831 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:58,831 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:58,833 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:58,833 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:58,835 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:58,835 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:58,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:58,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:58,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:58,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:58,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:58,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:58,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:58,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:58,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:58,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:58,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:58,990 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:58,991 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:58,992 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:58,993 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:58,994 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:58,994 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:58,995 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:04:59,405 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:04:59,405 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:05:02,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:02,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:02,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:02,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:02,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:02,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:02,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:02,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:02,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:02,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:02,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:02,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:02,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:02,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:02,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:02,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:02,206 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:05:03,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:03,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:03,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:03,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:03,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:03,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:03,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:03,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:03,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:03,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:03,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:03,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:03,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:03,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:03,577 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:03,577 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:04,207 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:05:04,609 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:04,609 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:04,612 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:04,613 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:04,614 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:04,614 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:04,615 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:04,616 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:04,620 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:04,620 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:04,621 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:04,622 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:04,623 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:04,624 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:04,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:04,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:04,837 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:04,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:04,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:04,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:04,842 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:04,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:04,844 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:04,844 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:04,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:04,849 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:04,850 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:04,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:04,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:04,853 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:04,853 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:04,854 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:06,610 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:06,610 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:06,614 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:06,614 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:06,615 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:06,615 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:06,616 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:06,617 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:06,621 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:06,621 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:06,623 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:06,623 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:06,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:06,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:06,626 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:06,626 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:07,504 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:07,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:07,508 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:07,508 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:07,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:07,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:07,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:07,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:07,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:07,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:07,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:07,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:07,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:07,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:07,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:07,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:08,209 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:05:09,476 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:09,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:09,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:09,481 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:09,481 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:09,482 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:09,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:09,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:09,487 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:09,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:09,489 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:09,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:09,491 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:09,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:09,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:09,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:10,210 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:05:11,716 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:11,716 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:11,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:11,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:11,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:11,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:11,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:11,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:11,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:11,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:11,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:11,728 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:11,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:11,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:11,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:11,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:11,975 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:11,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:11,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:11,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:11,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:11,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:11,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:11,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:11,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:11,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:11,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:11,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:11,990 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:11,990 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:11,991 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:11,991 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:12,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:12,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:12,958 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:12,959 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:12,959 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:12,960 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:12,961 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:12,961 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:12,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:12,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:12,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:12,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:12,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:12,969 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:12,970 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:12,970 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:13,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:13,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:13,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:13,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:13,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:13,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:13,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:13,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:13,156 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:13,157 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:13,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:13,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:13,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:13,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:13,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:13,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:13,925 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:05:14,212 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:05:14,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:14,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:14,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:14,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:14,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:14,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:14,680 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:14,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:14,684 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:14,684 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:14,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:14,687 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:14,688 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:14,688 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:14,689 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:14,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:14,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:05:14,706 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:05:15,606 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:15,606 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:15,609 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:15,610 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:15,610 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:15,611 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:15,612 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:15,613 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:15,616 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:15,616 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:15,618 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:15,618 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:15,620 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:15,620 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:15,621 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:15,621 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:15,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:15,940 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:15,944 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:15,944 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:15,945 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:15,945 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:15,946 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:15,947 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:15,950 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:15,951 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:15,952 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:15,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:15,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:15,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:15,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:15,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:16,213 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:05:17,024 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:17,024 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:17,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:17,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:17,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:17,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:17,033 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:17,033 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:17,033 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:17,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:17,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:17,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:17,037 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:17,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:17,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:17,039 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:18,214 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:05:20,215 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:05:20,661 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:20,661 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:20,664 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:20,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:20,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:20,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:20,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:20,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:20,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:20,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:20,672 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:20,672 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:20,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:20,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:20,675 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:20,676 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:20,682 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:20,683 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:20,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:20,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:20,687 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:20,687 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:20,688 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:20,689 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:20,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:20,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:20,693 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:20,694 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:20,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:20,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:20,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:20,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:23,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:23,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:23,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:23,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:23,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:23,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:23,165 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:23,165 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:23,168 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:23,169 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:23,170 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:23,170 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:23,172 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:23,172 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:23,173 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:23,173 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:24,216 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:05:24,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:24,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:24,247 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:24,247 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:24,248 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:24,248 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:24,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:24,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:24,253 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:24,253 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:24,255 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:24,255 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:24,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:24,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:24,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:24,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:24,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:24,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:24,508 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:24,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:24,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:24,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:24,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:24,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:24,514 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:24,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:24,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:24,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:24,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:24,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:24,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:24,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:26,217 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:05:26,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:26,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:26,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:26,759 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:26,759 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:26,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:26,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:26,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:26,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:26,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:26,765 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:26,766 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:26,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:26,768 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:26,768 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:26,769 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:26,834 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:26,835 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:26,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:26,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:26,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:26,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:26,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:26,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:26,844 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:26,844 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:26,845 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:26,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:26,847 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:26,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:26,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:26,849 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:27,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:27,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:27,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:27,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:27,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:27,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:27,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:27,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:27,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:27,569 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:27,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:27,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:27,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:27,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:27,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:27,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:27,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:27,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:27,583 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:27,584 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:27,584 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:27,585 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:27,586 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:27,586 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:27,589 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:27,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:27,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:27,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:27,592 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:27,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:27,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:27,594 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:28,218 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:05:28,707 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:28,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:28,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:28,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:28,713 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:28,713 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:28,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:28,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:28,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:28,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:28,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:28,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:28,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:28,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:28,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:28,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:29,048 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:29,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:29,052 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:29,052 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:29,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:29,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:29,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:29,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:29,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:29,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:29,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:29,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:29,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:29,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:29,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:29,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:30,033 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:05:30,033 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:05:30,219 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:05:31,092 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:31,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:31,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:31,097 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:31,097 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:31,098 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:31,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:31,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:31,103 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:31,103 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:31,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:31,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:31,107 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:31,107 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:31,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:31,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:31,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:31,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:31,157 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:31,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:31,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:31,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:31,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:31,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:31,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:31,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:31,165 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:31,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:31,167 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:31,168 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:31,168 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:31,169 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:32,220 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:05:32,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:32,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:32,254 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:32,254 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:32,255 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:32,255 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:32,256 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:32,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:32,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:32,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:32,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:32,262 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:32,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:32,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:32,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:32,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:33,463 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:33,464 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:33,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:33,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:33,468 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:33,469 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:33,470 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:33,470 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:33,473 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:33,473 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:33,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:33,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:33,476 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:33,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:33,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:33,478 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:34,221 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:05:36,221 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:05:38,222 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:05:39,388 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:39,388 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:39,391 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:39,391 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:39,392 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:39,392 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:39,393 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:39,394 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:39,396 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:39,396 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:39,397 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:39,398 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:39,399 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:39,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:39,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:39,401 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:40,223 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:05:42,224 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:05:42,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:42,314 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:42,316 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:42,317 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:42,317 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:42,318 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:42,319 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:42,319 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:42,321 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:42,322 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:42,323 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:42,323 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:42,324 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:42,325 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:42,326 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:42,326 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:43,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:43,375 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:43,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:43,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:43,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:43,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:43,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:43,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:43,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:43,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:43,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:43,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:43,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:43,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:43,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:43,388 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:44,000 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:05:44,225 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:05:45,213 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:45,214 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:45,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:45,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:45,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:45,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:45,218 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:45,219 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:45,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:45,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:45,223 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:45,223 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:45,224 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:45,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:45,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:45,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:45,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:05:45,268 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:05:46,227 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:05:48,228 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:05:50,229 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:05:51,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:51,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:51,142 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:51,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:51,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:51,144 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:51,145 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:51,145 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:51,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:51,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:51,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:51,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:51,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:51,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:51,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:51,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:52,230 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:05:53,021 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:53,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:53,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:53,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:53,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:53,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:53,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:53,028 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:53,029 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:53,030 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:53,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:53,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:53,033 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:53,033 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:53,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:53,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:05:54,231 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:05:56,232 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:05:58,232 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:06:00,233 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:06:00,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:06:00,641 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:06:02,234 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:06:04,235 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:06:06,236 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:06:08,236 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:06:10,237 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:06:12,238 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:06:12,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:12,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:12,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:12,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:12,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:12,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:12,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:12,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:12,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:12,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:12,388 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:12,388 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:12,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:12,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:12,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:12,391 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:13,213 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:13,213 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:13,215 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:13,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:13,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:13,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:13,218 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:13,218 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:13,219 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:13,220 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:13,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:13,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:13,222 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:13,223 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:13,223 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:13,224 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:14,076 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:06:14,181 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:14,181 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:14,184 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:14,184 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:14,185 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:14,185 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:14,186 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:14,189 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:14,190 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:14,190 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:14,191 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:14,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:14,193 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:14,193 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:14,194 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:14,194 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:14,239 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:06:14,393 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:14,394 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:14,395 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:14,396 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:14,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:14,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:14,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:14,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:14,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:14,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:14,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:14,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:14,455 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:14,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:14,457 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:14,457 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:14,458 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:14,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:14,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:14,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:14,555 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:14,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:14,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:14,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:14,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:14,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:14,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:14,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:14,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:14,816 DEBUG SenderThread:3760704 [sender.py:send():232] send: history +2022-08-30 19:06:15,053 DEBUG SenderThread:3760704 [sender.py:send():232] send: summary +2022-08-30 19:06:15,123 INFO SenderThread:3760704 [sender.py:_save_file():946] saving file wandb-summary.json with policy end +2022-08-30 19:06:15,239 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/wandb-summary.json +2022-08-30 19:06:15,860 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:06:15,861 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:06:16,240 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:06:17,240 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:06:22,242 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:06:26,191 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:26,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:26,195 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:26,196 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:26,197 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:26,197 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:26,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:26,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:26,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:26,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:26,207 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:26,208 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:26,209 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:26,210 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:26,211 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:26,211 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:26,244 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:06:29,996 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:29,996 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:30,000 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:30,001 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:30,001 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:30,002 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:30,003 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:30,004 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:30,009 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:30,010 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:30,012 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:30,012 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:30,014 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:30,014 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:30,015 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:30,015 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:30,246 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:06:31,203 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:06:31,203 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:06:31,296 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:31,296 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:31,300 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:31,300 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:31,301 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:31,301 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:31,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:31,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:31,308 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:31,308 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:31,310 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:31,311 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:31,312 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:31,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:31,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:31,314 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:31,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:31,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:31,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:31,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:31,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:31,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:31,513 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:31,513 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:31,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:31,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:31,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:31,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:31,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:31,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:31,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:31,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:31,934 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:31,935 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:31,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:31,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:31,940 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:31,941 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:31,942 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:31,943 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:31,948 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:31,948 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:31,950 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:31,950 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:31,952 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:31,952 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:31,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:31,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:34,248 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:06:38,249 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:06:40,250 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:06:44,198 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:06:44,252 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:06:46,253 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:06:46,433 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:06:46,433 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:06:49,398 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:49,399 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:49,402 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:49,402 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:49,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:49,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:49,404 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:49,405 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:49,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:49,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:49,411 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:49,411 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:49,413 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:49,413 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:49,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:49,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:50,254 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:06:50,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:50,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:50,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:50,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:50,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:50,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:50,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:50,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:50,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:50,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:50,664 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:50,664 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:50,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:50,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:50,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:50,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:52,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:52,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:52,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:52,039 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:52,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:52,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:52,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:52,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:52,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:52,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:52,048 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:52,048 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:52,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:52,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:52,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:52,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:52,255 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:06:56,165 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:56,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:56,169 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:56,169 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:56,170 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:56,170 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:56,171 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:56,172 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:56,175 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:56,176 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:56,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:56,178 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:56,179 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:56,179 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:56,180 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:56,180 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:56,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:56,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:56,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:56,203 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:56,204 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:56,204 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:56,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:56,206 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:56,209 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:56,209 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:56,211 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:56,211 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:56,212 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:56,213 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:56,214 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:56,214 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:56,256 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:06:57,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:57,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:57,831 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:57,832 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:57,832 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:57,833 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:57,834 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:57,834 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:57,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:57,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:57,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:57,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:57,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:57,842 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:57,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:57,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:58,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:58,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:58,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:58,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:58,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:58,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:58,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:58,043 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:58,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:58,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:58,048 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:58,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:58,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:58,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:58,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:58,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:58,257 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:06:59,321 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:59,322 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:59,325 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:59,326 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:59,327 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:59,327 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:59,328 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:59,329 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:59,332 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:59,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:59,334 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:59,335 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:59,336 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:59,336 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:59,337 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:06:59,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:00,258 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:07:00,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:00,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:00,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:00,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:00,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:00,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:00,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:00,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:00,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:00,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:00,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:00,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:00,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:00,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:00,583 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:00,584 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:01,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:07:01,655 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:07:02,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:02,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:02,993 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:02,993 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:02,994 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:02,994 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:02,996 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:02,996 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:03,000 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:03,000 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:03,001 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:03,002 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:03,003 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:03,004 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:03,005 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:03,005 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:03,170 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:03,170 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:03,174 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:03,174 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:03,175 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:03,175 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:03,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:03,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:03,181 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:03,181 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:03,183 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:03,183 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:03,185 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:03,185 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:03,186 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:03,186 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:03,328 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:03,329 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:03,332 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:03,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:03,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:03,334 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:03,335 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:03,336 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:03,339 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:03,339 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:03,341 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:03,341 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:03,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:03,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:03,344 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:03,345 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:04,260 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:07:04,588 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:04,588 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:04,592 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:04,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:04,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:04,594 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:04,595 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:04,596 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:04,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:04,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:04,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:04,602 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:04,603 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:04,604 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:04,604 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:04,605 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:05,552 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:05,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:05,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:05,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:05,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:05,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:05,558 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:05,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:05,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:05,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:05,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:05,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:05,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:05,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:05,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:05,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:05,596 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:05,596 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:05,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:05,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:05,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:05,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:05,602 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:05,602 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:05,606 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:05,606 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:05,608 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:05,608 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:05,610 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:05,610 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:05,611 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:05,611 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:06,261 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:07:06,950 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:06,950 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:06,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:06,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:06,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:06,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:06,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:06,957 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:06,960 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:06,961 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:06,962 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:06,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:06,964 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:06,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:06,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:06,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:08,262 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:07:09,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:09,443 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:09,447 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:09,447 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:09,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:09,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:09,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:09,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:09,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:09,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:09,455 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:09,455 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:09,457 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:09,457 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:09,458 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:09,458 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:10,263 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:07:10,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:10,513 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:10,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:10,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:10,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:10,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:10,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:10,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:10,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:10,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:10,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:10,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:10,525 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:10,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:10,527 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:10,527 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:12,264 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:07:12,785 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:12,785 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:12,788 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:12,788 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:12,789 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:12,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:12,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:12,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:12,794 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:12,795 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:12,796 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:12,796 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:12,797 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:12,798 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:12,799 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:12,799 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:12,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:12,837 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:12,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:12,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:12,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:12,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:12,842 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:12,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:12,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:12,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:12,847 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:12,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:12,849 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:12,850 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:12,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:12,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:14,275 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:07:15,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:15,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:15,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:15,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:15,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:15,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:15,127 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:15,127 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:15,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:15,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:15,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:15,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:15,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:15,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:15,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:15,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,118 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,127 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,271 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:07:16,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,391 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,391 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,392 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,393 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,396 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,396 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,398 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,398 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,401 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,401 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,408 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,412 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,412 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,413 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,413 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,418 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,421 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,489 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,494 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,498 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,498 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,514 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,514 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:16,934 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:07:16,934 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:07:18,272 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:07:18,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:18,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:18,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:18,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:18,375 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:18,375 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:18,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:18,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:18,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:18,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:18,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:18,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:18,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:18,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:18,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:18,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:18,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:18,653 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:18,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:18,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:18,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:18,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:18,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:18,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:18,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:18,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:18,664 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:18,664 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:18,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:18,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:18,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:18,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:20,273 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:07:21,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:21,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:21,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:21,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:21,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:21,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:21,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:21,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:21,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:21,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:21,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:21,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:21,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:21,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:21,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:21,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:21,862 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:21,862 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:21,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:21,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:21,866 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:21,867 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:21,868 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:21,868 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:21,871 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:21,871 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:21,873 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:21,873 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:21,874 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:21,875 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:21,876 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:21,876 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:22,274 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:07:23,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:23,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:23,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:23,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:23,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:23,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:23,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:23,732 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:23,734 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:23,735 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:23,736 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:23,737 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:23,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:23,739 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:23,740 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:23,740 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:24,773 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:24,774 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:24,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:24,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:24,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:24,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:24,779 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:24,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:24,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:24,783 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:24,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:24,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:24,785 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:24,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:24,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:24,787 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:24,869 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:24,869 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:24,872 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:24,872 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:24,873 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:24,873 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:24,874 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:24,875 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:24,877 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:24,878 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:24,879 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:24,879 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:24,880 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:24,881 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:24,881 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:24,882 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:25,275 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:07:27,276 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:07:27,745 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:27,746 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:27,748 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:27,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:27,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:27,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:27,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:27,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:27,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:27,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:27,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:27,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:27,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:27,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:27,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:27,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:29,278 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:07:31,279 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:07:32,176 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:07:32,176 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:07:32,447 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:32,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:32,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:32,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:32,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:32,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:32,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:32,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:32,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:32,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:32,457 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:32,458 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:32,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:32,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:32,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:32,461 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:32,489 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:32,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:32,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:32,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:32,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:32,494 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:32,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:32,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:32,498 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:32,498 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:32,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:32,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:32,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:32,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:32,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:32,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:33,280 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:07:35,281 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:07:36,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:36,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:36,855 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:36,855 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:36,856 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:36,856 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:36,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:36,858 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:36,860 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:36,860 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:36,861 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:36,862 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:36,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:36,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:36,864 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:36,864 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:37,281 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:07:39,282 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:07:41,283 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:07:42,233 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:42,234 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:42,236 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:42,237 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:42,237 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:42,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:42,239 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:42,239 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:42,241 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:42,241 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:42,242 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:42,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:42,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:42,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:42,245 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:42,246 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:43,284 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:07:44,348 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:07:45,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:45,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:45,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:45,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:45,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:45,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:45,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:45,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:45,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:45,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:45,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:45,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:45,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:45,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:45,272 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:45,272 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:45,285 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:07:47,286 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:07:47,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:07:47,474 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:07:49,287 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:07:51,288 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:07:53,289 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:07:55,290 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:07:57,291 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:07:59,292 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:07:59,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:59,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:59,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:59,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:59,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:59,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:59,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:59,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:59,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:59,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:59,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:59,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:59,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:59,725 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:59,725 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:07:59,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:00,842 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:00,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:00,844 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:00,844 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:01,008 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:01,012 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:01,013 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:01,014 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:01,014 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:01,015 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:01,015 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:01,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:01,293 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:08:02,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:08:02,709 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:08:03,294 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:08:03,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:03,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:03,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:03,513 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:03,513 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:03,514 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:03,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:03,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:03,525 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:03,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:03,528 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:03,528 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:03,530 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:03,531 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:03,531 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:03,532 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:05,176 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:05,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:05,180 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:05,181 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:05,182 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:05,182 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:05,184 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:05,184 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:05,191 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:05,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:05,193 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:05,194 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:05,196 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:05,196 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:05,197 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:05,197 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:09,297 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:08:09,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:09,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:09,569 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:09,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:09,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:09,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:09,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:09,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:09,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:09,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:09,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:09,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:09,583 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:09,583 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:09,584 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:09,584 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:13,299 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:08:14,420 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:08:15,300 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:08:18,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:08:18,072 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:08:19,301 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:08:23,303 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:08:25,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:25,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:25,514 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:25,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:25,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:25,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:25,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:25,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:25,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:25,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:25,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:25,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:25,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:25,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:25,527 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:25,528 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:25,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:25,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:25,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:25,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:25,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:25,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:25,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:25,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:25,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:25,710 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:25,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:25,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:25,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:25,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:25,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:25,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:25,871 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:25,872 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:25,876 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:25,876 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:25,877 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:25,877 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:25,879 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:25,879 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:25,884 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:25,884 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:25,886 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:25,886 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:25,887 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:25,888 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:25,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:25,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:26,090 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:26,090 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:26,095 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:26,095 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:26,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:26,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:26,098 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:26,098 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:26,102 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:26,103 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:26,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:26,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:26,106 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:26,107 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:26,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:26,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:27,305 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:08:27,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:27,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:27,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:27,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:27,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:27,375 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:27,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:27,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:27,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:27,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:27,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:27,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:27,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:27,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:27,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:27,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:29,306 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:08:33,308 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:08:33,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:08:33,376 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:08:35,309 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:08:38,938 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:38,938 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:38,941 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:38,942 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:38,942 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:38,943 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:38,944 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:38,945 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:38,949 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:38,949 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:38,950 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:38,951 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:38,952 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:38,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:38,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:38,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:39,311 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:08:41,312 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:08:43,313 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:08:44,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:44,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:44,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:44,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:44,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:44,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:44,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:44,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:44,037 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:44,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:44,039 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:44,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:44,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:44,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:44,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:44,043 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:44,188 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:44,188 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:44,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:44,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:44,193 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:44,193 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:44,194 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:44,195 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:44,198 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:44,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:44,200 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:44,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:44,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:44,203 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:44,203 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:44,204 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:44,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:44,478 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:44,481 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:44,481 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:44,482 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:44,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:44,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:44,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:44,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:44,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:44,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:44,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:44,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:44,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:44,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:44,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:44,496 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:08:46,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:46,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:46,725 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:46,725 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:46,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:46,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:46,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:46,728 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:46,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:46,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:46,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:46,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:46,735 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:46,735 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:46,736 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:46,736 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:46,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:46,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:46,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:46,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:46,759 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:46,759 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:46,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:46,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:46,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:46,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:46,766 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:46,766 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:46,768 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:46,768 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:46,769 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:46,769 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:47,315 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:08:48,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:48,496 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:48,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:48,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:48,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:48,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:48,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:48,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:48,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:48,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:48,508 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:48,508 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:48,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:48,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:48,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:48,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:48,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:08:48,702 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:08:49,316 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:08:49,429 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:49,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:49,433 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:49,433 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:49,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:49,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:49,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:49,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:49,439 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:49,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:49,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:49,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:49,443 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:49,443 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:49,444 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:49,444 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:50,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:50,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:50,991 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:50,992 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:50,992 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:50,993 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:50,994 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:50,995 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:50,998 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:50,998 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:51,000 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:51,000 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:51,002 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:51,002 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:51,003 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:51,003 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:51,317 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:08:52,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:52,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:52,052 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:52,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:52,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:52,054 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:52,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:52,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:52,059 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:52,059 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:52,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:52,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:52,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:52,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:52,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:52,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:52,082 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:52,082 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:52,085 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:52,086 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:52,087 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:52,087 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:52,088 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:52,089 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:52,092 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:52,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:52,094 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:52,094 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:52,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:52,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:52,097 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:52,097 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:53,223 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:53,224 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:53,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:53,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:53,228 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:53,229 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:53,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:53,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:53,234 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:53,234 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:53,235 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:53,236 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:53,237 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:53,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:53,239 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:53,239 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:53,318 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:08:54,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:54,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:54,358 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:54,359 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:54,359 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:54,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:54,361 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:54,362 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:54,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:54,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:54,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:54,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:54,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:54,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:54,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:54,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:54,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:54,728 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:54,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:54,732 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:54,732 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:54,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:54,734 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:54,735 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:54,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:54,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:54,740 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:54,740 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:54,742 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:54,742 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:54,743 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:54,744 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:56,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:56,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:56,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:56,785 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:56,785 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:56,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:56,787 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:56,788 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:56,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:56,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:56,792 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:56,793 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:56,794 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:56,794 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:56,795 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:56,796 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:57,319 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:08:59,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:59,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:59,044 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:59,044 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:59,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:59,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:59,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:59,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:59,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:59,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:59,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:59,052 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:59,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:59,054 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:59,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:59,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:59,320 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:08:59,331 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:59,332 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:59,335 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:59,335 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:59,336 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:59,336 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:59,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:59,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:59,341 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:59,341 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:59,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:59,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:59,345 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:59,345 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:59,346 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:08:59,346 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:01,321 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:09:02,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:02,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:02,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:02,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:02,725 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:02,725 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:02,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:02,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:02,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:02,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:02,732 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:02,732 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:02,734 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:02,734 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:02,735 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:02,736 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:03,322 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:09:03,867 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:03,868 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:03,871 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:03,872 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:03,872 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:03,873 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:03,874 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:03,875 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:03,877 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:03,878 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:03,879 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:03,880 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:03,881 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:03,882 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:03,882 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:03,883 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:03,946 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:09:03,946 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:09:04,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:04,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:04,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:04,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:04,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:04,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:04,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:04,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:04,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:04,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:04,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:04,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:04,728 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:04,728 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:04,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:04,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:04,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:04,842 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:04,845 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:04,845 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:04,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:04,847 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:04,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:04,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:04,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:04,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:04,853 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:04,854 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:04,855 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:04,855 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:04,856 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:04,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:04,882 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:04,883 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:04,885 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:04,886 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:04,886 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:04,887 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:04,888 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:04,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:04,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:04,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:04,893 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:04,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:04,895 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:04,896 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:04,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:04,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:05,009 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:05,010 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:05,013 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:05,014 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:05,014 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:05,015 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:05,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:05,017 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:05,020 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:05,020 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:05,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:05,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:05,024 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:05,024 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:05,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:05,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:05,323 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:09:07,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:07,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:07,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:07,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:07,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:07,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:07,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:07,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:07,157 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:07,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:07,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:07,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:07,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:07,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:07,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:07,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:07,324 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:09:07,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:07,957 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:07,959 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:07,960 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:07,961 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:07,961 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:07,962 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:07,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:07,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:07,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:07,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:07,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:07,969 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:07,970 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:07,970 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:07,971 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:08,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:08,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:08,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:08,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:08,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:08,141 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:08,142 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:08,142 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:08,145 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:08,145 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:08,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:08,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:08,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:08,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:08,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:08,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:09,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:09,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:09,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:09,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:09,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:09,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:09,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:09,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:09,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:09,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:09,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:09,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:09,138 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:09,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:09,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:09,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:09,325 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:09:09,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:09,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:09,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:09,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:09,375 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:09,375 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:09,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:09,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:09,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:09,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:09,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:09,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:09,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:09,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:09,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:09,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:10,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:10,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:10,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:10,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:10,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:10,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:10,073 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:10,073 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:10,075 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:10,076 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:10,077 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:10,077 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:10,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:10,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:10,080 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:10,080 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:11,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:11,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:11,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:11,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:11,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:11,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:11,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:11,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:11,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:11,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:11,990 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:11,991 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:11,992 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:11,993 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:11,993 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:11,994 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:12,245 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:12,245 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:12,248 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:12,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:12,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:12,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:12,251 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:12,251 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:12,254 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:12,254 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:12,255 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:12,256 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:12,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:12,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:12,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:12,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:13,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:13,127 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:13,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:13,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:13,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:13,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:13,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:13,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:13,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:13,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:13,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:13,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:13,138 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:13,138 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:13,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:13,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:13,327 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:09:14,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:14,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:14,220 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:14,220 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:14,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:14,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:14,222 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:14,223 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:14,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:14,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:14,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:14,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:14,228 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:14,229 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:14,229 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:14,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:14,569 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:09:15,328 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:09:17,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:17,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:17,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:17,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:17,097 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:17,097 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:17,098 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:17,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:17,101 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:17,102 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:17,103 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:17,103 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:17,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:17,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:17,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:17,106 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:17,329 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:09:19,311 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:09:19,312 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:09:19,330 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:09:21,330 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:09:22,653 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:22,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:22,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:22,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:22,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:22,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:22,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:22,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:22,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:22,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:22,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:22,664 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:22,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:22,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:22,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:22,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:22,694 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:22,694 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:22,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:22,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:22,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:22,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:22,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:22,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:22,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:22,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:22,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:22,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:22,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:22,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:22,707 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:22,707 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:23,331 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:09:25,332 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:09:26,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:26,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:26,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:26,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:26,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:26,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:26,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:26,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:26,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:26,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:26,759 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:26,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:26,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:26,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:26,762 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:26,762 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:27,333 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:09:29,059 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:29,059 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:29,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:29,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:29,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:29,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:29,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:29,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:29,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:29,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:29,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:29,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:29,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:29,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:29,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:29,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:29,334 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:09:31,335 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:09:33,335 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:09:33,661 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:33,661 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:33,664 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:33,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:33,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:33,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:33,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:33,668 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:33,669 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:33,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:33,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:33,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:33,672 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:33,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:33,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:33,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:34,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:09:34,567 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:09:35,336 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:09:37,337 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:09:39,338 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:09:41,339 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:09:43,340 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:09:44,642 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:09:45,341 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:09:46,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:46,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:46,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:46,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:46,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:46,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:46,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:46,725 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:46,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:46,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:46,728 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:46,728 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:46,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:46,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:46,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:46,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:47,476 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:47,476 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:47,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:47,478 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:47,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:47,644 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:47,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:47,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:47,646 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:47,647 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:47,647 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:47,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:47,813 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:47,899 DEBUG SenderThread:3760704 [sender.py:send():232] send: history +2022-08-30 19:09:48,076 DEBUG SenderThread:3760704 [sender.py:send():232] send: summary +2022-08-30 19:09:48,147 INFO SenderThread:3760704 [sender.py:_save_file():946] saving file wandb-summary.json with policy end +2022-08-30 19:09:48,342 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:09:48,342 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/wandb-summary.json +2022-08-30 19:09:49,789 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:09:49,790 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:09:50,343 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:09:54,345 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:09:54,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:54,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:54,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:54,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:54,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:54,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:54,759 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:54,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:54,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:54,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:54,769 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:54,769 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:54,771 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:54,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:54,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:54,773 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:56,210 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:56,210 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:56,214 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:56,214 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:56,215 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:56,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:56,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:56,218 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:56,224 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:56,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:56,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:56,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:56,228 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:56,229 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:56,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:09:56,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:00,347 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:10:02,253 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:02,254 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:02,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:02,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:02,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:02,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:02,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:02,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:02,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:02,267 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:02,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:02,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:02,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:02,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:02,272 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:02,272 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:02,348 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:10:05,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:10:05,144 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:10:06,350 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:10:10,352 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:10:14,354 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:10:14,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:14,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:14,585 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:14,586 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:14,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:14,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:14,589 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:14,589 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:14,594 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:14,594 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:14,596 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:14,596 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:14,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:14,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:14,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:14,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:14,729 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:10:16,354 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:10:17,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:17,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:17,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:17,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:17,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:17,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:17,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:17,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:17,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:17,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:17,392 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:17,392 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:17,394 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:17,394 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:17,395 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:17,396 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:18,807 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:18,807 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:18,811 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:18,811 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:18,812 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:18,812 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:18,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:18,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:18,818 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:18,818 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:18,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:18,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:18,822 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:18,822 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:18,823 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:18,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:20,356 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:10:20,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:10:20,383 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:10:20,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:20,508 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:20,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:20,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:20,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:20,513 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:20,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:20,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:20,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:20,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:20,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:20,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:20,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:20,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:20,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:20,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:22,357 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:10:24,862 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:24,862 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:24,866 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:24,866 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:24,867 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:24,868 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:24,869 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:24,869 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:24,873 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:24,874 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:24,875 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:24,876 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:24,877 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:24,878 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:24,878 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:24,879 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:26,358 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:10:28,359 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:10:30,360 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:10:33,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:33,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:33,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:33,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:33,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:33,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:33,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:33,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:33,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:33,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:33,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:33,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:33,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:33,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:33,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:33,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:34,362 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:10:35,433 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:35,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:35,437 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:35,438 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:35,439 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:35,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:35,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:35,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:35,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:35,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:35,447 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:35,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:35,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:35,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:35,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:35,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:35,604 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:10:35,605 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:10:35,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:35,853 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:35,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:35,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:35,858 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:35,858 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:35,859 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:35,860 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:35,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:35,864 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:35,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:35,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:35,867 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:35,867 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:35,868 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:35,869 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:36,363 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:10:37,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:37,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:37,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:37,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:37,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:37,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:37,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:37,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:37,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:37,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:37,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:37,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:37,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:37,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:37,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:37,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:38,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:38,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:38,272 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:38,273 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:38,273 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:38,274 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:38,275 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:38,276 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:38,279 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:38,279 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:38,281 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:38,281 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:38,283 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:38,283 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:38,284 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:38,284 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:38,364 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:10:39,336 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:39,337 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:39,340 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:39,341 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:39,341 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:39,342 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:39,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:39,344 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:39,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:39,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:39,349 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:39,349 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:39,351 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:39,351 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:39,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:39,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:40,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:40,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:40,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:40,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:40,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:40,592 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:40,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:40,594 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:40,597 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:40,597 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:40,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:40,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:40,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:40,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:40,602 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:40,602 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:42,366 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:10:44,366 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:10:44,806 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:10:45,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:45,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:45,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:45,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:45,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:45,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:45,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:45,569 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:45,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:45,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:45,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:45,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:45,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:45,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:45,577 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:45,577 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:45,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:45,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:45,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:45,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:45,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:45,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:45,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:45,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:45,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:45,728 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:45,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:45,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:45,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:45,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:45,732 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:45,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:46,367 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:10:46,476 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:46,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:46,479 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:46,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:46,481 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:46,481 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:46,482 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:46,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:46,486 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:46,486 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:46,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:46,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:46,489 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:46,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:46,491 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:46,491 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:47,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:47,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:47,775 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:47,776 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:47,776 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:47,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:47,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:47,779 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:47,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:47,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:47,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:47,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:47,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:47,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:47,787 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:47,787 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:47,902 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:47,903 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:47,906 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:47,906 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:47,907 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:47,907 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:47,908 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:47,909 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:47,912 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:47,912 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:47,913 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:47,914 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:47,915 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:47,916 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:47,916 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:47,917 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:48,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:48,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:48,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:48,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:48,138 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:48,138 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:48,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:48,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:48,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:48,144 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:48,145 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:48,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:48,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:48,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:48,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:48,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:48,368 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:10:48,842 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:48,842 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:48,845 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:48,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:48,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:48,847 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:48,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:48,849 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:48,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:48,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:48,853 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:48,854 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:48,856 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:48,856 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:48,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:48,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:49,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:49,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:49,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:49,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:49,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:49,028 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:49,029 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:49,029 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:49,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:49,033 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:49,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:49,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:49,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:49,037 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:49,037 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:49,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:49,184 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:49,184 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:49,188 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:49,188 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:49,189 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:49,189 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:49,191 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:49,191 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:49,194 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:49,194 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:49,196 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:49,196 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:49,198 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:49,198 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:49,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:49,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:50,369 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:10:50,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:10:50,829 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:10:52,370 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:10:53,291 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:53,291 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:53,294 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:53,295 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:53,295 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:53,296 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:53,297 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:53,297 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:53,300 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:53,301 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:53,302 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:53,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:53,304 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:53,304 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:53,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:53,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:54,371 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:10:54,586 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:54,586 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:54,589 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:54,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:54,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:54,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:54,592 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:54,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:54,595 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:54,596 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:54,597 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:54,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:54,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:54,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:54,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:54,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:54,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:54,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:54,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:54,649 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:54,649 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:54,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:54,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:54,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:54,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:54,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:54,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:54,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:54,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:54,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:54,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:54,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:56,974 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:56,975 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:56,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:56,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:56,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:56,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:56,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:56,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:56,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:56,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:56,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:56,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:56,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:56,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:56,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:56,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:57,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:57,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:57,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:57,898 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:57,898 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:57,899 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:57,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:57,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:57,903 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:57,904 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:57,905 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:57,905 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:57,906 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:57,907 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:57,908 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:57,908 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:58,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:58,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:58,020 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:58,020 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:58,021 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:58,021 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:58,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:58,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:58,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:58,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:58,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:58,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:58,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:58,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:58,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:58,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:58,372 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:10:59,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:59,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:59,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:59,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:59,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:59,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:59,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:59,127 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:59,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:59,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:59,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:59,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:59,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:59,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:59,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:10:59,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:00,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:00,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:00,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:00,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:00,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:00,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:00,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:00,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:00,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:00,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:00,073 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:00,073 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:00,074 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:00,075 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:00,076 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:00,076 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:00,373 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:11:00,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:00,890 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:00,893 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:00,893 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:00,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:00,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:00,895 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:00,896 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:00,898 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:00,899 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:00,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:00,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:00,901 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:00,902 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:00,903 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:00,903 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:01,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:01,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:01,224 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:01,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:01,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:01,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:01,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:01,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:01,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:01,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:01,231 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:01,232 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:01,233 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:01,233 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:01,234 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:01,235 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:02,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:02,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:02,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:02,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:02,141 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:02,141 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:02,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:02,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:02,145 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:02,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:02,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:02,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:02,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:02,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:02,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:02,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:02,374 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:11:02,975 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:02,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:02,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:02,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:02,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:02,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:02,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:02,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:02,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:02,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:02,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:02,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:02,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:02,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:02,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:02,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:03,830 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:03,830 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:03,833 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:03,834 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:03,834 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:03,835 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:03,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:03,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:03,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:03,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:03,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:03,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:03,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:03,842 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:03,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:03,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:04,018 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:04,018 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:04,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:04,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:04,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:04,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:04,024 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:04,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:04,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:04,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:04,029 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:04,029 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:04,030 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:04,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:04,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:04,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:04,375 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:11:04,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:04,893 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:04,895 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:04,896 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:04,896 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:04,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:04,898 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:04,898 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:04,901 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:04,901 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:04,902 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:04,903 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:04,904 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:04,904 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:04,905 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:04,906 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:04,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:04,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:04,970 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:04,970 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:04,971 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:04,971 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:04,972 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:04,973 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:04,975 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:04,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:04,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:04,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:04,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:04,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:04,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:04,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:06,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:11:06,050 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:11:06,375 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:11:07,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:07,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:07,694 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:07,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:07,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:07,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:07,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:07,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:07,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:07,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:07,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:07,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:07,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:07,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:07,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:07,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:07,779 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:07,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:07,783 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:07,783 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:07,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:07,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:07,785 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:07,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:07,788 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:07,788 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:07,789 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:07,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:07,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:07,792 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:07,792 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:07,793 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:08,376 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:11:10,377 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:11:12,378 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:11:14,030 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:14,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:14,033 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:14,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:14,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:14,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:14,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:14,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:14,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:14,039 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:14,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:14,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:14,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:14,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:14,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:14,043 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:14,378 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:11:14,884 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:11:15,619 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:15,619 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:15,622 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:15,622 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:15,623 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:15,623 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:15,624 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:15,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:15,627 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:15,627 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:15,628 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:15,629 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:15,630 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:15,630 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:15,631 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:15,631 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:16,294 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:16,294 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:16,297 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:16,297 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:16,298 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:16,298 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:16,299 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:16,300 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:16,302 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:16,302 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:16,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:16,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:16,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:16,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:16,306 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:16,306 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:16,379 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:11:18,380 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:11:19,240 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:19,241 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:19,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:19,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:19,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:19,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:19,245 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:19,246 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:19,248 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:19,248 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:19,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:19,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:19,251 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:19,251 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:19,252 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:19,252 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:20,381 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:11:21,289 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:11:21,290 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:11:22,382 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:11:24,383 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:11:25,923 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:25,924 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:25,926 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:25,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:25,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:25,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:25,929 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:25,929 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:25,931 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:25,931 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:25,932 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:25,933 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:25,934 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:25,935 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:25,935 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:25,936 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:26,383 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:11:28,384 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:11:30,385 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:11:32,386 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:11:34,387 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:11:35,085 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:35,085 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:35,086 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:35,087 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:35,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:35,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:35,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:35,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:35,245 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:35,245 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:35,246 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:35,246 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:36,388 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:11:36,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:11:36,506 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:11:40,358 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:40,359 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:40,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:40,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:40,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:40,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:40,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:40,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:40,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:40,375 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:40,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:40,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:40,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:40,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:40,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:40,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:42,391 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:11:44,960 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:11:46,393 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:11:49,763 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:49,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:49,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:49,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:49,768 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:49,769 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:49,770 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:49,771 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:49,775 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:49,776 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:49,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:49,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:49,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:49,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:49,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:49,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:50,256 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:50,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:50,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:50,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:50,262 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:50,262 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:50,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:50,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:50,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:50,270 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:50,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:50,272 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:50,273 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:50,274 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:50,275 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:50,275 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:50,394 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:11:51,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:11:51,727 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:11:54,396 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:11:56,397 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:11:56,826 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:56,827 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:56,830 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:56,831 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:56,832 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:56,832 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:56,833 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:56,834 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:56,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:56,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:56,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:56,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:56,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:56,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:56,844 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:11:56,844 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:00,399 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:12:04,401 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:12:06,402 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:12:07,024 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:12:07,024 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:12:09,558 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:09,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:09,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:09,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:09,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:09,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:09,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:09,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:09,569 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:09,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:09,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:09,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:09,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:09,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:09,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:09,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:09,743 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:09,744 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:09,748 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:09,748 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:09,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:09,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:09,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:09,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:09,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:09,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:09,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:09,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:09,759 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:09,759 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:09,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:09,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:11,404 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:12:13,405 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:12:13,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:13,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:13,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:13,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:13,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:13,713 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:13,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:13,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:13,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:13,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:13,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:13,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:13,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:13,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:13,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:13,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:15,037 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:12:15,406 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:12:19,332 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:19,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:19,336 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:19,337 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:19,337 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:19,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:19,339 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:19,339 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:19,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:19,344 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:19,345 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:19,346 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:19,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:19,348 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:19,348 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:19,349 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:19,408 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:12:21,408 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:12:22,255 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:12:22,256 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:12:23,653 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:23,653 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:23,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:23,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:23,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:23,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:23,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:23,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:23,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:23,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:23,664 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:23,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:23,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:23,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:23,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:23,668 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:23,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:23,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:23,734 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:23,734 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:23,735 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:23,736 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:23,737 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:23,737 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:23,740 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:23,741 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:23,742 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:23,743 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:23,744 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:23,745 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:23,745 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:23,746 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:25,410 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:12:27,411 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:12:28,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:28,592 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:28,595 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:28,596 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:28,597 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:28,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:28,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:28,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:28,604 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:28,605 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:28,606 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:28,607 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:28,608 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:28,609 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:28,610 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:28,610 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:29,412 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:12:30,214 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:30,215 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:30,218 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:30,219 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:30,219 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:30,220 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:30,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:30,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:30,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:30,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:30,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:30,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:30,228 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:30,229 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:30,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:30,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:30,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:30,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:30,262 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:30,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:30,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:30,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:30,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:30,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:30,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:30,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:30,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:30,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:30,273 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:30,273 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:30,274 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:30,274 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:33,414 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:12:33,615 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:33,615 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:33,618 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:33,619 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:33,619 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:33,620 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:33,621 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:33,621 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:33,624 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:33,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:33,626 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:33,627 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:33,628 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:33,628 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:33,629 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:33,630 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:34,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:34,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:34,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:34,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:34,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:34,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:34,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:34,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:34,759 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:34,759 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:34,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:34,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:34,762 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:34,763 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:34,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:34,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:35,415 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:12:35,798 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:35,799 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:35,802 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:35,802 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:35,803 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:35,803 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:35,804 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:35,805 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:35,808 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:35,808 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:35,809 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:35,810 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:35,811 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:35,812 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:35,812 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:35,813 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:37,110 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:37,111 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:37,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:37,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:37,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:37,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:37,117 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:37,117 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:37,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:37,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:37,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:37,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:37,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:37,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:37,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:37,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:37,416 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:12:37,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:12:37,483 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:12:39,417 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:12:39,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:39,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:39,498 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:39,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:39,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:39,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:39,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:39,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:39,504 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:39,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:39,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:39,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:39,508 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:39,508 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:39,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:39,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:40,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:40,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:40,788 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:40,788 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:40,789 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:40,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:40,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:40,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:40,794 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:40,795 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:40,796 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:40,797 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:40,799 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:40,799 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:40,800 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:40,800 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:41,418 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:12:41,479 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:41,479 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:41,482 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:41,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:41,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:41,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:41,485 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:41,485 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:41,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:41,489 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:41,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:41,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:41,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:41,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:41,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:41,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:42,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:42,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:42,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:42,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:42,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:42,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:42,783 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:42,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:42,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:42,787 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:42,788 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:42,788 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:42,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:42,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:42,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:42,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:42,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:42,844 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:42,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:42,847 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:42,847 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:42,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:42,849 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:42,850 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:42,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:42,853 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:42,854 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:42,855 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:42,856 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:42,856 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:42,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:42,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:43,419 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:12:43,616 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:43,617 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:43,619 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:43,620 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:43,620 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:43,621 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:43,622 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:43,622 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:43,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:43,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:43,627 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:43,627 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:43,628 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:43,629 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:43,629 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:43,630 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:43,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:43,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:43,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:43,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:43,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:43,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:43,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:43,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:43,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:43,664 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:43,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:43,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:43,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:43,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:43,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:43,668 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:43,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:43,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:43,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:43,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:43,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:43,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:43,707 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:43,707 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:43,710 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:43,710 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:43,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:43,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:43,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:43,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:43,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:43,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:44,011 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:44,011 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:44,015 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:44,015 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:44,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:44,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:44,017 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:44,018 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:44,021 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:44,021 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:44,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:44,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:44,024 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:44,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:44,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:44,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:44,950 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:44,951 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:44,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:44,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:44,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:44,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:44,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:44,957 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:44,960 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:44,960 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:44,961 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:44,962 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:44,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:44,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:44,964 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:44,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:45,115 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:12:45,420 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:12:46,969 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:46,969 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:46,972 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:46,972 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:46,973 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:46,973 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:46,974 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:46,975 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:46,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:46,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:46,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:46,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:46,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:46,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:46,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:46,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:47,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:47,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:47,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:47,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:47,059 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:47,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:47,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:47,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:47,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:47,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:47,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:47,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:47,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:47,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:47,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:47,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:47,141 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:47,142 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:47,145 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:47,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:47,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:47,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:47,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:47,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:47,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:47,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:47,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:47,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:47,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:47,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:47,155 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:47,156 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:49,422 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:12:50,075 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:50,076 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:50,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:50,080 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:50,080 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:50,081 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:50,082 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:50,082 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:50,085 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:50,085 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:50,086 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:50,087 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:50,088 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:50,089 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:50,089 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:50,090 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:50,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:50,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:50,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:50,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:50,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:50,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:50,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:50,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:50,155 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:50,156 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:50,157 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:50,157 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:50,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:50,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:50,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:50,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:51,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:51,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:51,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:51,203 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:51,203 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:51,204 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:51,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:51,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:51,207 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:51,208 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:51,209 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:51,209 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:51,211 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:51,211 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:51,212 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:51,212 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:51,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:51,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:51,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:51,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:51,262 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:51,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:51,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:51,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:51,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:51,267 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:51,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:51,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:51,270 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:51,270 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:51,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:51,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:51,423 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:12:51,871 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:51,871 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:51,874 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:51,874 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:51,875 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:51,875 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:51,876 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:51,877 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:51,879 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:51,879 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:51,881 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:51,881 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:51,882 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:51,883 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:51,883 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:51,884 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:52,744 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:12:52,744 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:12:53,424 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:12:53,739 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:53,740 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:53,742 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:53,743 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:53,744 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:53,744 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:53,746 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:53,746 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:53,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:53,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:53,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:53,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:53,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:53,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:53,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:53,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:53,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:53,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:53,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:53,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:53,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:53,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:53,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:53,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:53,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:53,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:53,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:53,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:53,990 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:53,991 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:53,991 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:53,992 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:54,013 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:54,013 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:54,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:54,017 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:54,017 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:54,018 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:54,019 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:54,019 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:54,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:54,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:54,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:54,024 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:54,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:54,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:54,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:54,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:55,425 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:12:55,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:55,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:55,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:55,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:55,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:55,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:55,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:55,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:55,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:55,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:55,710 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:55,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:55,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:55,713 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:55,713 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:55,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:55,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:55,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:55,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:55,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:55,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:55,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:55,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:55,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:55,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:55,759 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:55,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:55,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:55,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:55,762 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:55,763 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:55,763 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:56,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:56,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:56,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:56,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:56,716 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:56,716 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:56,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:56,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:56,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:56,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:56,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:56,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:56,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:56,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:56,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:56,725 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:57,426 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:12:59,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:59,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:59,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:59,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:59,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:59,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:59,392 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:59,392 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:59,394 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:59,395 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:59,396 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:59,396 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:59,397 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:59,398 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:59,398 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:59,399 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:12:59,427 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:13:01,428 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:13:02,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:02,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:02,117 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:02,117 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:02,117 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:02,118 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:02,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:02,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:02,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:02,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:02,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:02,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:02,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:02,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:02,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:02,127 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:03,429 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:13:03,743 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:03,743 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:03,746 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:03,746 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:03,747 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:03,747 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:03,748 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:03,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:03,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:03,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:03,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:03,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:03,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:03,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:03,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:03,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:05,430 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:13:06,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:06,675 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:06,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:06,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:06,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:06,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:06,680 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:06,680 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:06,682 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:06,683 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:06,684 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:06,684 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:06,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:06,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:06,687 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:06,687 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:06,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:06,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:06,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:06,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:06,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:06,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:06,716 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:06,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:06,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:06,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:06,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:06,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:06,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:06,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:06,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:06,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:07,431 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:13:07,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:13:07,969 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:13:09,432 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:13:11,433 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:13:12,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:12,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:12,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:12,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:12,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:12,602 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:12,603 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:12,603 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:12,605 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:12,605 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:12,606 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:12,607 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:12,608 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:12,608 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:12,609 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:12,609 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:13,434 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:13:14,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:14,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:14,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:14,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:14,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:14,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:14,728 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:14,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:14,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:14,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:14,732 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:14,732 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:14,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:14,734 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:14,734 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:14,735 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:15,194 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:13:15,435 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:13:17,436 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:13:19,437 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:13:21,438 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:13:23,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:23,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:23,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:23,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:23,176 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:23,181 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:23,182 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:23,182 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:23,183 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:23,184 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:23,184 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:23,184 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:23,189 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:13:23,190 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:13:23,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 19:13:23,474 DEBUG SenderThread:3760704 [sender.py:send():232] send: history +2022-08-30 19:13:23,562 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:13:23,667 DEBUG SenderThread:3760704 [sender.py:send():232] send: summary +2022-08-30 19:13:23,741 INFO SenderThread:3760704 [sender.py:_save_file():946] saving file wandb-summary.json with policy end +2022-08-30 19:13:24,536 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:13:24,537 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/wandb-summary.json +2022-08-30 19:13:25,537 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:13:27,537 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:13:29,538 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:13:31,539 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:13:33,540 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:13:35,541 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:13:37,542 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:13:38,538 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:13:38,540 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:13:39,544 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:13:41,545 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:13:43,546 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:13:45,317 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:13:45,547 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:13:47,548 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:13:49,548 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:13:51,549 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:13:53,550 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:13:53,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:13:53,756 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:13:55,551 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:13:57,552 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:13:59,553 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:14:01,554 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:14:03,555 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:14:05,556 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:14:07,557 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:14:08,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:14:08,977 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:14:09,558 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:14:11,559 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:14:14,560 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:14:15,377 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:14:16,561 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:14:18,562 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:14:20,562 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:14:22,563 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:14:24,214 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:14:24,214 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:14:24,564 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:14:26,565 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:14:28,565 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:14:30,566 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:14:32,567 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:14:34,567 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:14:36,568 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:14:38,569 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:14:39,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:14:39,441 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:14:40,570 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:14:42,571 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:14:44,572 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:14:45,448 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:14:46,573 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:14:48,574 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:14:50,575 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:14:52,575 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:14:54,576 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:14:54,688 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:14:54,688 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:14:56,577 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:14:58,578 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:15:00,579 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:15:02,580 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:15:04,580 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:15:06,581 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:15:08,582 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:15:09,907 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:15:09,907 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:15:10,583 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:15:12,584 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:15:14,585 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:15:15,522 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:15:16,586 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:15:18,586 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:15:20,587 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:15:22,588 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:15:24,588 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:15:25,222 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:15:25,222 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:15:26,589 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:15:28,590 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:15:30,591 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:15:32,591 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:15:34,592 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:15:36,593 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:15:38,594 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:15:40,537 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:15:40,538 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:15:41,595 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:15:43,596 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:15:45,590 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:15:45,597 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:15:47,597 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:15:49,598 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:15:51,599 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:15:53,600 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:15:55,601 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:15:55,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:15:55,780 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:15:57,602 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:15:59,603 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:16:01,604 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:16:03,605 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:16:05,606 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:16:07,608 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:16:09,609 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:16:11,003 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:16:11,003 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:16:11,610 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:16:13,611 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:16:15,612 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:16:15,662 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:16:17,613 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:16:19,613 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:16:21,614 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:16:23,615 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:16:25,616 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:16:26,222 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:16:26,223 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:16:27,617 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:16:29,617 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:16:31,618 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:16:33,619 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:16:35,620 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:16:37,620 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:16:39,621 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:16:41,437 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:16:41,438 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:16:41,622 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:16:43,623 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:16:45,624 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:16:45,736 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:16:47,625 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:16:49,626 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:16:51,627 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:16:53,628 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:16:55,629 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:16:56,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:16:56,711 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:16:57,630 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:16:59,631 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:17:01,632 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:17:03,633 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:17:05,634 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:17:07,635 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:17:09,636 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:17:11,637 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:17:11,935 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:17:11,935 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:17:13,638 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:17:15,639 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:17:15,810 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:17:17,640 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:17:19,640 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:17:22,642 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:17:24,643 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:17:26,644 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:17:27,215 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:17:27,215 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:17:28,645 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:17:30,645 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:17:32,646 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:17:34,647 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:17:36,648 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:17:38,649 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:17:40,650 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:17:42,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:17:42,477 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:17:42,651 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:17:44,652 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:17:45,882 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:17:46,653 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:17:48,654 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:17:50,655 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:17:52,656 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:17:54,657 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:17:56,658 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:17:57,770 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:17:57,771 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:17:58,659 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:18:00,660 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:18:02,661 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:18:04,662 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:18:06,663 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:18:08,664 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:18:10,665 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:18:12,666 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:18:13,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:18:13,023 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:18:14,667 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:18:15,953 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:18:16,668 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:18:18,668 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:18:20,669 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:18:22,670 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:18:24,671 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:18:26,671 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:18:28,253 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:18:28,254 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:18:28,672 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:18:30,673 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:18:32,674 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:18:34,675 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:18:36,676 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:18:38,676 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:18:40,677 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:18:42,678 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:18:43,469 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:18:43,470 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:18:44,679 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:18:46,029 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:18:46,680 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:18:48,681 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:18:50,682 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:18:52,682 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:18:54,683 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:18:56,684 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:18:58,685 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:18:58,860 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:18:58,860 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:19:00,686 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:19:03,687 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:19:05,688 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:19:07,689 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:19:09,690 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:19:11,691 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:19:13,692 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:19:14,080 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:19:14,081 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:19:15,693 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:19:16,108 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:19:17,694 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:19:19,695 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:19:21,696 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:19:23,697 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:19:25,698 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:19:27,698 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:19:29,294 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:19:29,294 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:19:29,699 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:19:31,700 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:19:33,701 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:19:35,702 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:19:37,703 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:19:39,704 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:19:41,705 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:19:43,706 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:19:44,683 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:19:44,684 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:19:45,707 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:19:46,182 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:19:47,708 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:19:49,708 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:19:51,710 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:19:53,710 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:19:55,711 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:19:57,712 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:19:59,713 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:19:59,909 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:19:59,909 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:20:01,714 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:20:03,715 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:20:05,715 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:20:07,716 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:20:09,717 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:20:11,718 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:20:13,718 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:20:15,188 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:20:15,188 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:20:15,719 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:20:16,260 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:20:17,720 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:20:19,720 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:20:21,721 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:20:23,722 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:20:25,723 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:20:28,724 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:20:30,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:20:30,445 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:20:30,725 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:20:32,726 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:20:34,727 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:20:36,728 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:20:38,729 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:20:40,730 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:20:42,731 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:20:44,732 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:20:45,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:20:45,839 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:20:46,334 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:20:46,733 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:20:48,734 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:20:50,735 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:20:52,736 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:20:54,737 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:20:56,739 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:20:58,739 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:21:00,740 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:21:01,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:21:01,064 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:21:02,741 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:21:04,741 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:21:06,742 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:21:08,743 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:21:10,744 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:21:12,744 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:21:14,745 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:21:16,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:21:16,360 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:21:16,583 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:21:16,746 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:21:18,747 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:21:20,748 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:21:22,749 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:21:24,750 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:21:26,751 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:21:28,752 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:21:30,753 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:21:31,618 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:21:31,619 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:21:32,754 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:21:34,755 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:21:36,755 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:21:38,756 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:21:40,757 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:21:42,758 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:21:44,759 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:21:46,483 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:21:46,760 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:21:46,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:21:46,839 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:21:48,760 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:21:50,761 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:21:52,762 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:21:55,763 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:21:57,764 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:21:59,765 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:22:01,766 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:22:02,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:22:02,149 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:22:03,767 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:22:05,768 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:22:07,769 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:22:09,770 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:22:11,770 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:22:13,772 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:22:15,772 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:22:16,561 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:22:17,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:22:17,467 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:22:17,773 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:22:19,774 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:22:21,775 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:22:23,776 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:22:25,777 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:22:27,777 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:22:29,778 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:22:31,779 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:22:32,744 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:22:32,744 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:22:33,780 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:22:35,781 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:22:37,782 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:22:39,783 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:22:41,783 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:22:43,784 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:22:45,785 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:22:46,632 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:22:47,786 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:22:47,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:22:47,968 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:22:49,787 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:22:51,788 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:22:53,789 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:22:55,790 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:22:57,791 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:22:59,792 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:23:01,792 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:23:03,193 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:23:03,194 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:23:03,793 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:23:05,794 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:23:07,796 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:23:09,797 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:23:11,798 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:23:14,799 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:23:16,698 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:23:16,800 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:23:18,552 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:23:18,553 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:23:18,801 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:23:20,802 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:23:22,803 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:23:24,804 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:23:26,805 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:23:28,806 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:23:30,806 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:23:32,807 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:23:33,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:23:33,839 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:23:34,808 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:23:36,809 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:23:38,810 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:23:40,811 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:23:42,812 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:23:44,812 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:23:46,777 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:23:46,814 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:23:48,814 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:23:49,106 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:23:49,107 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:23:50,815 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:23:52,816 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:23:54,816 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:23:56,817 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:23:58,818 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:24:00,819 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:24:02,819 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:24:04,348 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:24:04,348 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:24:04,820 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:24:06,821 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:24:08,823 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:24:10,823 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:24:12,824 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:24:14,825 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:24:16,826 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:24:16,853 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:24:18,827 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:24:19,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:24:19,569 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:24:20,828 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:24:22,829 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:24:24,830 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:24:26,830 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:24:28,831 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:24:30,832 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:24:32,833 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:24:34,833 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:24:34,845 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:24:34,845 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:24:36,834 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:24:39,836 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:24:41,837 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:24:43,838 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:24:45,839 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:24:46,925 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:24:47,840 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:24:49,841 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:24:50,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:24:50,067 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:24:51,842 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:24:53,843 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:24:55,844 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:24:57,845 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:24:59,846 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:25:01,847 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:25:03,848 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:25:05,290 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:25:05,291 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:25:05,849 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:25:07,850 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:25:09,851 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:25:11,852 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:25:13,853 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:25:15,854 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:25:17,001 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:25:17,855 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:25:19,856 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:25:20,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:25:20,524 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:25:21,857 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:25:23,858 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:25:25,859 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:25:27,860 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:25:29,861 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:25:31,862 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:25:33,863 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:25:35,739 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:25:35,740 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:25:35,864 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:25:37,865 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:25:39,866 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:25:41,867 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:25:43,868 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:25:45,869 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:25:47,077 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:25:47,870 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:25:49,871 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:25:51,017 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:25:51,017 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:25:51,872 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:25:53,873 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:25:55,874 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:25:57,875 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:25:59,876 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:26:01,877 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:26:03,878 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:26:05,879 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:26:06,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:26:06,366 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:26:07,880 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:26:09,881 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:26:11,882 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:26:14,883 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:26:16,884 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:26:17,153 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:26:18,885 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:26:20,886 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:26:21,589 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:26:21,590 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:26:22,887 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:26:24,888 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:26:26,889 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:26:28,890 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:26:30,891 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:26:32,891 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:26:34,892 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:26:36,893 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:26:36,925 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:26:36,925 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:26:38,894 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:26:40,895 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:26:42,896 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:26:44,896 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:26:46,897 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:26:47,226 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:26:48,898 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:26:50,899 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:26:52,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:26:52,260 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:26:52,900 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:26:54,900 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:26:56,901 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:26:58,902 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:27:00,903 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:27:02,904 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:27:04,904 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:27:06,905 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:27:07,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:27:07,480 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:27:08,906 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:27:10,907 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:27:12,907 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:27:14,908 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:27:16,909 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:27:17,301 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:27:19,910 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:27:21,911 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:27:22,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:27:22,699 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:27:23,912 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:27:25,913 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:27:27,913 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:27:29,914 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:27:31,915 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:27:33,916 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:27:35,917 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:27:37,917 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:27:37,918 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:27:37,921 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:27:39,922 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:27:41,923 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:27:43,924 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:27:45,924 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:27:47,376 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:27:47,925 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:27:49,926 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:27:51,927 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:27:53,206 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:27:53,207 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:27:53,928 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:27:55,929 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:27:57,929 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:27:59,930 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:28:01,931 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:28:03,932 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:28:05,933 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:28:07,934 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:28:08,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:28:08,425 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:28:09,935 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:28:11,936 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:28:13,937 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:28:15,938 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:28:17,449 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:28:17,939 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:28:19,939 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:28:21,940 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:28:23,649 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:28:23,649 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:28:23,941 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:28:25,942 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:28:27,943 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:28:29,943 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:28:31,944 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:28:33,945 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:28:35,946 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:28:37,947 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:28:38,864 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:28:38,864 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:28:39,948 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:28:42,949 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:28:44,950 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:28:46,951 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:28:47,526 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:28:48,952 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:28:50,953 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:28:52,954 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:28:54,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:28:54,244 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:28:54,955 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:28:56,956 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:28:58,957 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:29:00,958 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:29:02,959 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:29:04,960 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:29:06,961 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:29:08,962 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:29:09,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:29:09,477 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:29:10,963 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:29:12,964 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:29:14,965 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:29:16,966 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:29:17,604 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:29:18,966 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:29:20,967 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:29:22,968 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:29:24,798 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:29:24,798 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:29:24,969 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:29:26,970 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:29:28,971 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:29:30,972 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:29:32,973 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:29:34,974 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:29:36,974 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:29:38,975 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:29:40,019 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:29:40,020 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:29:40,977 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:29:42,977 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:29:44,978 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:29:46,979 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:29:47,678 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:29:48,980 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:29:50,980 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:29:52,981 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:29:54,982 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:29:55,242 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:29:55,242 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:29:56,982 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:29:58,983 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:30:00,984 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:30:02,984 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:30:04,985 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:30:06,986 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:30:08,987 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:30:10,525 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:30:10,525 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:30:10,988 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:30:12,988 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:30:14,989 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:30:16,990 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:30:17,754 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:30:18,991 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:30:20,992 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:30:22,993 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:30:25,741 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:30:25,741 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:30:25,994 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:30:27,995 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:30:29,996 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:30:31,997 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:30:33,998 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:30:35,999 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:30:38,000 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:30:40,001 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:30:41,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:30:41,125 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:30:42,002 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:30:44,003 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:30:46,004 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:30:47,830 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:30:48,005 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:30:50,005 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:30:52,006 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:30:54,007 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:30:56,008 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:30:56,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:30:56,453 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:30:58,009 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:31:00,010 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:31:02,011 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:31:04,012 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:31:06,013 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:31:08,014 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:31:10,015 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:31:11,687 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:31:11,687 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:31:12,016 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:31:14,017 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:31:16,017 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:31:17,906 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:31:18,018 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:31:20,019 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:31:22,020 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:31:24,021 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:31:26,021 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:31:26,908 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:31:26,909 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:31:28,022 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:31:30,023 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:31:32,023 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:31:34,024 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:31:36,025 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:31:38,026 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:31:40,027 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:31:42,028 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:31:42,213 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:31:42,214 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:31:44,029 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:31:46,030 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:31:47,983 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:31:48,030 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:31:50,031 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:31:52,032 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:31:54,033 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:31:57,034 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:31:57,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:31:57,484 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:31:59,036 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:32:01,036 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:32:03,037 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:32:05,038 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:32:07,039 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:32:09,040 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:32:11,041 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:32:12,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:32:12,778 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:32:13,042 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:32:15,043 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:32:17,044 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:32:18,059 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:32:19,045 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:32:21,046 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:32:23,047 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:32:25,047 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:32:27,048 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:32:28,044 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:32:28,044 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:32:29,049 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:32:31,050 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:32:33,051 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:32:35,052 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:32:37,053 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:32:39,054 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:32:42,055 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:32:43,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:32:43,267 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:32:44,056 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:32:46,057 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:32:48,057 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:32:48,132 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:32:50,058 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:32:52,059 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:32:54,060 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:32:56,061 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:32:58,062 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:32:58,617 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:32:58,617 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:33:00,063 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:33:02,064 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:33:04,064 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:33:06,065 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:33:08,066 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:33:10,067 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:33:12,068 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:33:13,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:33:13,928 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:33:14,069 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:33:16,070 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:33:18,071 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:33:18,207 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:33:20,072 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:33:22,073 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:33:24,074 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:33:26,075 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:33:28,075 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:33:29,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:33:29,151 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:33:30,076 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:33:32,077 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:33:34,078 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:33:36,079 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:33:38,080 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:33:40,081 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:33:42,081 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:33:44,082 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:33:44,542 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:33:44,543 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:33:46,083 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:33:48,084 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:33:48,288 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:33:50,085 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:33:52,086 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:33:54,087 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:33:56,088 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:33:58,088 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:33:59,788 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:33:59,788 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:34:00,089 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:34:02,090 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:34:05,091 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:34:07,092 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:34:09,093 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:34:11,094 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:34:13,095 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:34:15,081 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:34:15,081 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:34:15,095 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:34:17,096 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:34:18,381 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:34:19,097 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:34:21,098 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:34:23,099 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:34:25,099 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:34:27,101 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:34:29,101 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:34:30,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:34:30,423 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:34:31,102 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:34:33,104 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:34:35,105 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:34:37,105 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:34:39,106 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:34:41,107 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:34:43,108 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:34:45,109 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:34:45,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:34:45,648 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:34:47,110 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:34:48,435 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:34:49,111 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:34:51,112 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:34:53,113 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:34:55,114 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:34:57,115 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:34:59,116 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:35:01,008 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:35:01,008 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:35:01,116 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:35:03,117 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:35:05,118 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:35:07,119 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:35:09,120 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:35:11,121 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:35:13,122 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:35:15,123 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:35:16,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:35:16,230 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:35:17,124 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:35:18,509 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:35:19,125 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:35:21,126 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:35:24,127 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:35:26,128 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:35:28,129 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:35:30,130 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:35:31,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:35:31,565 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:35:32,131 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:35:34,132 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:35:36,133 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:35:38,134 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:35:40,135 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:35:42,136 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:35:44,137 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:35:46,137 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:35:46,788 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:35:46,788 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:35:48,138 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:35:48,585 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:35:50,139 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:35:52,139 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:35:54,140 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:35:56,141 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:35:58,142 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:36:00,143 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:36:02,074 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:36:02,075 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:36:02,144 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:36:04,144 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:36:06,145 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:36:08,146 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:36:10,146 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:36:12,147 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:36:14,148 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:36:16,149 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:36:17,297 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:36:17,297 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:36:18,150 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:36:18,655 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:36:20,151 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:36:22,152 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:36:24,152 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:36:26,153 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:36:28,154 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:36:30,155 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:36:32,156 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:36:32,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:36:32,513 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:36:34,157 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:36:36,158 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:36:38,159 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:36:40,160 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:36:42,161 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:36:44,162 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:36:46,163 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:36:47,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:36:47,897 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:36:48,164 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:36:48,731 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:36:50,165 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:36:52,165 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:36:54,166 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:36:56,167 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:36:58,168 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:37:00,169 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:37:02,170 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:37:03,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:37:03,132 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:37:04,170 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:37:06,171 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:37:08,172 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:37:11,173 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:37:13,173 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:37:15,174 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:37:17,175 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:37:18,350 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:37:18,350 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:37:18,805 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:37:19,176 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:37:21,177 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:37:23,177 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:37:25,178 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:37:27,179 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:37:29,180 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:37:31,181 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:37:33,182 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:37:33,624 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:37:33,625 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:37:35,182 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:37:37,183 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:37:39,184 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:37:41,184 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:37:43,185 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:37:45,186 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:37:47,187 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:37:48,882 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:37:48,908 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:37:48,908 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:37:49,188 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:37:51,188 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:37:53,189 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:37:55,190 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:37:57,191 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:37:59,192 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:38:01,193 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:38:03,194 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:38:04,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:38:04,121 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:38:05,195 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:38:07,196 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:38:09,197 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:38:11,198 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:38:13,199 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:38:15,200 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:38:17,201 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:38:18,958 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:38:19,202 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:38:19,461 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:38:19,462 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:38:21,202 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:38:23,203 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:38:25,204 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:38:27,205 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:38:29,206 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:38:31,207 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:38:33,208 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:38:34,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:38:34,765 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:38:36,209 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:38:38,210 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:38:40,211 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:38:42,212 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:38:44,212 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:38:46,213 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:38:48,214 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:38:49,038 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:38:50,014 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:38:50,015 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:38:50,215 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:38:52,216 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:38:54,217 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:38:56,217 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:38:58,218 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:39:00,218 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:39:02,219 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:39:04,220 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:39:05,282 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:39:05,283 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:39:06,221 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:39:08,221 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:39:10,222 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:39:12,223 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:39:14,224 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:39:16,224 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:39:18,225 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:39:19,114 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:39:20,226 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:39:20,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:39:20,510 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:39:22,226 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:39:24,227 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:39:26,228 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:39:28,229 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:39:30,230 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:39:32,231 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:39:34,232 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:39:35,817 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:39:35,817 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:39:36,232 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:39:38,233 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:39:40,234 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:39:42,235 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:39:44,236 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:39:46,237 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:39:48,237 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:39:49,192 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:39:50,238 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:39:52,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:39:52,205 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:39:52,239 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:39:54,240 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:39:56,241 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:39:58,242 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:40:00,243 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:40:02,243 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:40:04,244 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:40:06,245 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:40:07,608 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:40:07,608 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:40:09,246 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:40:11,247 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:40:13,248 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:40:15,249 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:40:17,250 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:40:19,250 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:40:19,270 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:40:21,251 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:40:22,829 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:40:22,829 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:40:23,252 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:40:25,252 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:40:27,253 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:40:29,254 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:40:31,254 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:40:33,255 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:40:35,256 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:40:37,257 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:40:38,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:40:38,052 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:40:39,258 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:40:41,258 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:40:43,259 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:40:45,260 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:40:47,261 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:40:49,261 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:40:49,346 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:40:51,262 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:40:53,263 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:40:53,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:40:53,378 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:40:55,264 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:40:57,265 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:40:59,266 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:41:01,266 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:41:03,267 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:41:05,268 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:41:07,268 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:41:08,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:41:08,600 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:41:09,269 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:41:11,270 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:41:13,270 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:41:15,271 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:41:17,272 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:41:19,273 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:41:19,421 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:41:21,274 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:41:23,274 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:41:23,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:41:23,840 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:41:25,275 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:41:27,276 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:41:29,277 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:41:31,277 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:41:33,278 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:41:35,279 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:41:37,280 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:41:39,090 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:41:39,090 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:41:39,281 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:41:41,282 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:41:43,283 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:41:45,284 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:41:47,285 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:41:49,496 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:41:50,286 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:41:52,287 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:41:54,288 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:41:54,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:41:54,441 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:41:56,289 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:41:58,290 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:42:00,291 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:42:02,291 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:42:04,292 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:42:06,293 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:42:08,294 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:42:09,747 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:42:09,747 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:42:10,295 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:42:12,296 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:42:14,297 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:42:16,298 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:42:18,298 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:42:19,570 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:42:20,299 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:42:22,300 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:42:24,301 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:42:25,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:42:25,032 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:42:26,302 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:42:28,302 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:42:30,303 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:42:32,304 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:42:34,305 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:42:36,306 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:42:38,307 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:42:40,256 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:42:40,256 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:42:40,308 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:42:42,308 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:42:44,309 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:42:46,310 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:42:48,311 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:42:49,643 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:42:50,312 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:42:52,313 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:42:54,313 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:42:55,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:42:55,659 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:42:56,314 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:42:58,315 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:43:00,315 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:43:02,316 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:43:04,317 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:43:06,318 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:43:08,319 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:43:10,319 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:43:10,877 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:43:10,877 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:43:12,320 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:43:14,321 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:43:16,322 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:43:18,323 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:43:19,715 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:43:20,323 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:43:22,324 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:43:24,325 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:43:26,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:43:26,119 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:43:26,325 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:43:28,326 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:43:30,327 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:43:32,328 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:43:34,328 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:43:36,329 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:43:38,330 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:43:40,330 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:43:41,356 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:43:41,356 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:43:42,331 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:43:44,332 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:43:46,333 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:43:49,334 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:43:49,781 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:43:51,335 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:43:53,335 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:43:55,336 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:43:56,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:43:56,572 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:43:57,337 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:43:59,338 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:44:01,339 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:44:03,340 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:44:05,341 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:44:07,341 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:44:09,342 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:44:11,343 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:44:11,849 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:44:11,849 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:44:13,344 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:44:15,345 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:44:17,345 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:44:19,346 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:44:19,855 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:44:21,347 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:44:23,348 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:44:25,349 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:44:27,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:44:27,152 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:44:27,349 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:44:29,351 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:44:31,351 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:44:33,352 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:44:35,353 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:44:37,354 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:44:39,354 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:44:41,356 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:44:42,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:44:42,372 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:44:43,356 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:44:45,357 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:44:47,358 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:44:49,359 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:44:49,931 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:44:51,368 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:44:53,365 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:44:55,366 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:44:57,367 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:44:57,707 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:44:57,707 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:44:59,368 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:45:01,368 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:45:03,369 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:45:05,370 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:45:07,371 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:45:09,372 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:45:11,373 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:45:13,374 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:45:14,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:45:14,559 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:45:15,375 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:45:17,376 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:45:19,377 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:45:20,008 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:45:21,378 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:45:23,379 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:45:25,380 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:45:27,380 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:45:29,381 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:45:29,932 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:45:29,932 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:45:31,382 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:45:33,383 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:45:35,384 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:45:37,385 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:45:39,385 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:45:41,386 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:45:43,387 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:45:45,222 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:45:45,222 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:45:46,389 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:45:48,390 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:45:50,082 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:45:50,391 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:45:52,392 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:45:54,393 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:45:56,393 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:45:58,394 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:46:00,395 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:46:00,531 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:46:00,532 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:46:02,397 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:46:04,397 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:46:06,398 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:46:08,399 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:46:10,399 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:46:12,400 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:46:14,401 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:46:15,823 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:46:15,824 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:46:16,402 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:46:18,403 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:46:20,156 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:46:20,404 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:46:22,405 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:46:24,406 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:46:26,407 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:46:28,407 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:46:30,408 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:46:31,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:46:31,047 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:46:32,409 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:46:34,410 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:46:36,411 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:46:38,412 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:46:40,413 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:46:42,414 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:46:44,415 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:46:46,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:46:46,365 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:46:46,416 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:46:48,417 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:46:50,223 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:46:50,418 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:46:52,419 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:46:54,420 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:46:56,421 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:46:58,422 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:47:00,423 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:47:01,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:47:01,760 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:47:02,424 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:47:04,425 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:47:06,425 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:47:08,426 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:47:10,427 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:47:12,428 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:47:14,429 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:47:16,430 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:47:16,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:47:16,980 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:47:18,431 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:47:20,295 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:47:20,431 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:47:22,432 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:47:24,433 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:47:26,434 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:47:28,435 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:47:30,436 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:47:32,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:47:32,201 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:47:32,436 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:47:35,437 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:47:37,438 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:47:39,439 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:47:41,440 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:47:43,441 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:47:45,442 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:47:47,443 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:47:47,540 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:47:47,540 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:47:49,444 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:47:50,370 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:47:51,445 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:47:53,446 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:47:55,447 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:47:57,448 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:47:59,448 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:48:01,449 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:48:02,835 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:48:02,835 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:48:03,450 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:48:05,451 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:48:07,452 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:48:09,453 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:48:11,454 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:48:13,455 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:48:15,456 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:48:17,456 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:48:18,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:48:18,049 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:48:19,457 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:48:20,435 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:48:21,458 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:48:23,459 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:48:25,460 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:48:27,460 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:48:29,461 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:48:31,462 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:48:33,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:48:33,339 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:48:33,463 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:48:35,464 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:48:37,464 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:48:39,465 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:48:41,466 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:48:43,467 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:48:45,468 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:48:47,468 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:48:48,623 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:48:48,623 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:48:49,469 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:48:50,511 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:48:51,470 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:48:54,472 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:48:56,472 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:48:58,473 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:49:00,474 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:49:02,475 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:49:03,861 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:49:03,861 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:49:04,476 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:49:06,477 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:49:08,478 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:49:10,478 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:49:12,479 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:49:14,480 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:49:16,481 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:49:18,482 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:49:19,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:49:19,238 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:49:20,482 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:49:20,579 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:49:22,483 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:49:24,484 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:49:26,485 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:49:28,486 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:49:30,487 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:49:32,487 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:49:34,488 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:49:34,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:49:34,564 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:49:36,489 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:49:38,490 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:49:40,490 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:49:42,491 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:49:44,491 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:49:46,492 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:49:48,493 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:49:49,885 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:49:49,885 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:49:50,493 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:49:50,655 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:49:52,494 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:49:54,494 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:49:56,495 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:49:58,496 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:50:00,496 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:50:02,497 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:50:04,497 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:50:05,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:50:05,147 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:50:06,498 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:50:08,499 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:50:10,499 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:50:12,500 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:50:14,500 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:50:16,501 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:50:18,502 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:50:20,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:50:20,365 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:50:20,503 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:50:20,730 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:50:23,504 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:50:25,505 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:50:27,506 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:50:29,507 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:50:31,507 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:50:33,508 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:50:35,509 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:50:35,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:50:35,757 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:50:37,510 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:50:39,510 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:50:41,511 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:50:43,512 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:50:45,513 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:50:47,513 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:50:49,514 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:50:50,805 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:50:51,039 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:50:51,039 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:50:51,515 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:50:53,516 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:50:55,517 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:50:57,518 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:50:59,519 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:51:01,520 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:51:03,521 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:51:05,521 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:51:06,289 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:51:06,289 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:51:07,522 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:51:09,523 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:51:11,524 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:51:13,525 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:51:15,526 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:51:17,527 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:51:19,527 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:51:20,881 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:51:21,528 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:51:21,609 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:51:21,610 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:51:23,529 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:51:25,530 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:51:27,531 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:51:29,532 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:51:31,533 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:51:33,533 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:51:35,534 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:51:36,880 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:51:36,880 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:51:37,535 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:51:40,536 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:51:42,537 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:51:44,538 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:51:46,539 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:51:48,540 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:51:50,541 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:51:50,954 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:51:52,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:51:52,113 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:51:52,541 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:51:54,542 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:51:56,543 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:51:58,544 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:52:00,544 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:52:02,545 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:52:04,546 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:52:06,547 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:52:07,447 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:52:07,448 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:52:08,548 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:52:10,549 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:52:12,549 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:52:14,550 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:52:16,551 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:52:18,551 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:52:20,552 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:52:21,032 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:52:22,553 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:52:22,676 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:52:22,677 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:52:24,554 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:52:26,554 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:52:28,555 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:52:30,556 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:52:32,556 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:52:34,557 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:52:36,558 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:52:37,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:52:37,983 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:52:38,559 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:52:40,559 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:52:42,560 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:52:44,561 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:52:46,562 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:52:48,563 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:52:50,564 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:52:51,106 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:52:52,565 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:52:53,204 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:52:53,204 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:52:54,565 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:52:56,566 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:52:58,567 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:53:00,568 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:53:02,569 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:53:04,570 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:53:06,571 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:53:08,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:53:08,511 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:53:08,572 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:53:10,573 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:53:12,574 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:53:14,575 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:53:16,576 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:53:18,576 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:53:20,577 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:53:21,179 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:53:22,578 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:53:23,745 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:53:23,745 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:53:24,579 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:53:26,579 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:53:28,580 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:53:30,581 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:53:32,582 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:53:34,583 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:53:36,583 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:53:38,584 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:53:38,962 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:53:38,962 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:53:40,585 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:53:42,586 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:53:44,587 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:53:46,587 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:53:49,589 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:53:51,249 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:53:51,589 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:53:53,590 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:53:54,308 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:53:54,308 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:53:55,591 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:53:57,592 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:53:59,592 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:54:01,593 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:54:03,594 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:54:05,595 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:54:07,596 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:54:09,528 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:54:09,528 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:54:09,597 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:54:11,598 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:54:13,599 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:54:15,599 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:54:17,600 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:54:19,602 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:54:21,323 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:54:21,602 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:54:23,603 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:54:24,885 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:54:24,885 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:54:25,604 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:54:27,605 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:54:29,606 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:54:31,607 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:54:33,608 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:54:35,609 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:54:37,610 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:54:39,611 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:54:40,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:54:40,116 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:54:41,612 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:54:43,613 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:54:45,614 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:54:47,615 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:54:49,615 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:54:51,399 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:54:51,616 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:54:53,617 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:54:55,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:54:55,446 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:54:55,617 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:54:57,618 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:54:59,619 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:55:01,619 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:55:03,620 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:55:05,621 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:55:07,622 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:55:09,623 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:55:10,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:55:10,815 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:55:11,624 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:55:13,625 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:55:15,625 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:55:17,626 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:55:19,627 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:55:21,473 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:55:21,628 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:55:23,629 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:55:25,630 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:55:26,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:55:26,034 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:55:27,630 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:55:30,632 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:55:32,632 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:55:34,633 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:55:36,634 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:55:38,635 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:55:40,636 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:55:41,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:55:41,723 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:55:42,637 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:55:44,638 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:55:46,639 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:55:48,640 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:55:50,641 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:55:51,548 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:55:52,642 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:55:54,643 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:55:56,644 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:55:57,029 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:55:57,030 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:55:58,645 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:56:00,645 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:56:02,646 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:56:04,647 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:56:06,648 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:56:08,649 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:56:10,650 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:56:12,285 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:56:12,285 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:56:12,651 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:56:14,652 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:56:16,652 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:56:18,653 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:56:20,654 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:56:21,626 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:56:22,655 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:56:24,656 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:56:26,657 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:56:27,585 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:56:27,586 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:56:28,657 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:56:30,658 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:56:32,659 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:56:34,660 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:56:36,660 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:56:38,661 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:56:40,662 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:56:42,662 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:56:42,808 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:56:42,809 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:56:44,663 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:56:47,664 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:56:49,665 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:56:51,665 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:56:51,697 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:56:53,666 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:56:55,667 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:56:57,668 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:56:58,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:56:58,031 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:56:59,669 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:57:01,669 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:57:03,670 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:57:05,671 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:57:07,672 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:57:09,672 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:57:11,674 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:57:13,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:57:13,365 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:57:13,674 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:57:15,675 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:57:17,676 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:57:19,677 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:57:21,678 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:57:21,760 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:57:23,679 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:57:25,680 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:57:27,681 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:57:28,636 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:57:28,636 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:57:29,682 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:57:31,682 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:57:33,683 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:57:35,684 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:57:37,685 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:57:39,686 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:57:41,687 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:57:43,688 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:57:43,856 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:57:43,856 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:57:45,689 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:57:47,690 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:57:49,690 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:57:51,691 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:57:51,836 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:57:53,692 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:57:55,693 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:57:57,694 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:57:59,189 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:57:59,189 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:57:59,695 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:58:01,695 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:58:04,696 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:58:06,697 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:58:08,698 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:58:10,699 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:58:12,699 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:58:14,486 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:58:14,486 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:58:14,700 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:58:16,701 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:58:18,702 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:58:20,702 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:58:21,909 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:58:22,703 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:58:24,704 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:58:26,704 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:58:28,705 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:58:29,710 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:58:29,711 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:58:30,706 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:58:32,706 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:58:34,707 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:58:36,708 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:58:38,709 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:58:40,709 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:58:42,710 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:58:44,711 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:58:45,098 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:58:45,099 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:58:46,711 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:58:48,712 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:58:50,713 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:58:51,988 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:58:52,714 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:58:54,715 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:58:56,716 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:58:58,717 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:59:00,539 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:59:00,539 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:59:00,718 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:59:02,718 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:59:04,719 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:59:06,720 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:59:08,721 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:59:10,722 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:59:12,723 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:59:14,724 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:59:15,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:59:15,778 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:59:16,724 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:59:18,725 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:59:20,726 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:59:22,061 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:59:22,727 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:59:24,728 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:59:26,728 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:59:28,729 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:59:30,730 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:59:31,004 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:59:31,005 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:59:32,731 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:59:34,732 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:59:37,733 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:59:39,734 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:59:41,735 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:59:43,736 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:59:45,737 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:59:46,220 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 19:59:46,220 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 19:59:47,738 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:59:49,738 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:59:51,739 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:59:52,134 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 19:59:53,740 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:59:55,741 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:59:57,742 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 19:59:59,743 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:00:01,535 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:00:01,535 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:00:01,744 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:00:03,745 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:00:05,746 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:00:07,747 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:00:09,748 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:00:11,749 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:00:13,750 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:00:15,751 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:00:16,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:00:16,759 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:00:17,752 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:00:19,753 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:00:21,754 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:00:22,193 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 20:00:23,755 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:00:25,756 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:00:27,757 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:00:29,758 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:00:31,758 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:00:32,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:00:32,071 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:00:33,759 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:00:35,760 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:00:37,761 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:00:39,761 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:00:41,762 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:00:43,763 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:00:45,763 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:00:47,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:00:47,379 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:00:47,764 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:00:49,765 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:00:51,765 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:00:52,260 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 20:00:53,766 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:00:55,767 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:00:57,768 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:00:59,768 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:01:01,769 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:01:02,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:01:02,599 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:01:03,770 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:01:05,770 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:01:07,771 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:01:09,771 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:01:11,772 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:01:13,773 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:01:16,774 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:01:17,910 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:01:17,910 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:01:18,774 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:01:20,775 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:01:22,331 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 20:01:22,776 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:01:24,776 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:01:26,777 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:01:28,777 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:01:30,778 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:01:32,779 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:01:33,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:01:33,132 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:01:34,780 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:01:36,781 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:01:38,782 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:01:40,783 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:01:42,784 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:01:44,785 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:01:46,786 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:01:48,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:01:48,352 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:01:48,786 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:01:50,787 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:01:52,403 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 20:01:52,788 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:01:54,789 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:01:56,790 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:01:58,791 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:02:00,792 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:02:02,793 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:02:03,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:02:03,671 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:02:04,793 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:02:06,794 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:02:08,795 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:02:11,796 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:02:13,797 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:02:15,798 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:02:17,799 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:02:18,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:02:18,892 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:02:19,800 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:02:21,801 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:02:22,479 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 20:02:23,801 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:02:25,802 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:02:27,803 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:02:29,803 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:02:31,804 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:02:33,805 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:02:34,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:02:34,113 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:02:35,805 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:02:37,806 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:02:39,807 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:02:41,808 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:02:43,808 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:02:45,809 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:02:47,810 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:02:49,466 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:02:49,467 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:02:49,811 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:02:51,811 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:02:52,553 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 20:02:53,812 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:02:55,813 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:02:57,814 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:02:59,815 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:03:01,815 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:03:03,816 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:03:04,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:03:04,691 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:03:05,817 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:03:07,818 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:03:09,818 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:03:11,819 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:03:13,820 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:03:15,821 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:03:17,822 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:03:19,822 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:03:19,916 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:03:19,916 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:03:21,823 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:03:22,630 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 20:03:23,824 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:03:25,825 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:03:27,825 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:03:29,826 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:03:31,827 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:03:33,828 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:03:35,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:03:35,131 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:03:35,829 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:03:37,829 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:03:39,830 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:03:41,831 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:03:43,831 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:03:45,832 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:03:47,833 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:03:49,834 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:03:50,346 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:03:50,346 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:03:51,835 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:03:52,705 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 20:03:53,835 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:03:55,836 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:03:57,837 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:03:59,838 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:04:01,838 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:04:03,839 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:04:05,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:04:05,699 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:04:06,840 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:04:08,841 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:04:10,842 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:04:12,843 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:04:14,844 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:04:16,845 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:04:18,845 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:04:20,846 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:04:20,922 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:04:20,923 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:04:22,779 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 20:04:22,847 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:04:24,848 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:04:26,849 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:04:28,850 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:04:30,850 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:04:32,851 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:04:34,852 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:04:36,211 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:04:36,211 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:04:36,853 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:04:38,854 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:04:40,855 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:04:42,856 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:04:44,857 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:04:46,858 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:04:48,859 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:04:50,859 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:04:51,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:04:51,553 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:04:52,855 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 20:04:52,860 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:04:54,861 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:04:56,862 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:04:58,862 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:05:00,863 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:05:02,864 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:05:04,865 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:05:06,823 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:05:06,824 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:05:06,866 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:05:08,866 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:05:10,867 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:05:12,868 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:05:14,869 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:05:16,870 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:05:18,871 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:05:20,872 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:05:22,092 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:05:22,093 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:05:22,872 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:05:22,927 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 20:05:24,873 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:05:26,874 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:05:28,875 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:05:30,875 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:05:32,876 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:05:34,877 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:05:36,877 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:05:37,312 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:05:37,312 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:05:38,878 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:05:40,879 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:05:42,880 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:05:44,880 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:05:46,881 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:05:48,882 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:05:50,883 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:05:52,554 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:05:52,554 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:05:52,884 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:05:53,000 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 20:05:54,885 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:05:56,886 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:05:58,887 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:06:00,888 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:06:03,889 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:06:05,890 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:06:07,774 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:06:07,774 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:06:07,891 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:06:09,892 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:06:11,893 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:06:13,894 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:06:15,895 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:06:17,896 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:06:19,897 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:06:21,898 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:06:23,067 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 20:06:23,077 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:06:23,077 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:06:23,898 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:06:25,899 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:06:27,900 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:06:29,901 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:06:31,901 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:06:33,902 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:06:35,903 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:06:37,904 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:06:38,330 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:06:38,331 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:06:39,905 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:06:41,906 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:06:43,907 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:06:45,908 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:06:47,909 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:06:49,910 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:06:51,910 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:06:53,142 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 20:06:53,554 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:06:53,555 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:06:53,911 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:06:55,912 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:06:57,912 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:06:59,913 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:07:01,914 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:07:03,915 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:07:05,916 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:07:07,916 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:07:08,866 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:07:08,866 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:07:09,917 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:07:11,918 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:07:13,919 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:07:15,920 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:07:17,921 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:07:19,922 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:07:21,923 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:07:23,218 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 20:07:23,923 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:07:24,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:07:24,146 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:07:25,924 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:07:27,925 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:07:29,926 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:07:31,926 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:07:33,927 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:07:35,928 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:07:37,929 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:07:39,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:07:39,454 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:07:39,930 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:07:42,931 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:07:44,932 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:07:46,933 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:07:48,933 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:07:50,934 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:07:52,935 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:07:53,295 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 20:07:54,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:07:54,738 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:07:54,936 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:07:56,937 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:07:58,938 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:08:00,939 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:08:02,940 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:08:04,941 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:08:06,942 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:08:08,943 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:08:09,999 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:08:09,999 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:08:10,944 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:08:12,945 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:08:14,946 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:08:16,946 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:08:18,947 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:08:20,948 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:08:22,949 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:08:23,374 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 20:08:24,950 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:08:25,274 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:08:25,275 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:08:26,951 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:08:28,951 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:08:30,952 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:08:32,953 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:08:34,954 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:08:36,955 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:08:38,956 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:08:40,533 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:08:40,533 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:08:40,957 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:08:42,958 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:08:44,959 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:08:47,960 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:08:49,961 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:08:51,962 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:08:53,452 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 20:08:53,962 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:08:55,829 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:08:55,829 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:08:55,963 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:08:57,964 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:08:59,965 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:09:01,966 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:09:03,967 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:09:05,968 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:09:07,969 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:09:09,970 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:09:11,971 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:09:13,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:09:13,263 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:09:13,972 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:09:15,973 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:09:17,974 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:09:19,975 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:09:21,976 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:09:23,527 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 20:09:23,977 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:09:25,978 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:09:27,979 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:09:28,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:09:28,567 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:09:29,980 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:09:31,980 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:09:33,981 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:09:35,982 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:09:37,983 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:09:39,984 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:09:41,985 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:09:43,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:09:43,783 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:09:43,986 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:09:45,987 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:09:47,987 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:09:50,988 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:09:52,990 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:09:53,607 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 20:09:54,990 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:09:56,991 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:09:58,992 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:09:59,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:09:59,026 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:10:00,993 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:10:02,994 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:10:04,995 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:10:06,996 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:10:08,997 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:10:10,997 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:10:12,998 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:10:14,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:10:14,244 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:10:14,999 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:10:17,000 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:10:19,001 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:10:21,001 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:10:23,002 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:10:23,682 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 20:10:25,003 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:10:27,004 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:10:29,004 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:10:29,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:10:29,591 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:10:31,005 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:10:33,006 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:10:35,007 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:10:37,008 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:10:39,008 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:10:41,009 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:10:43,010 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:10:44,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:10:44,852 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:10:45,011 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:10:47,011 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:10:49,012 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:10:51,014 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:10:53,014 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:10:53,757 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 20:10:55,016 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:10:57,017 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:10:59,018 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:11:00,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:11:00,136 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:11:01,019 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:11:03,020 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:11:05,020 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:11:07,021 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:11:09,022 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:11:11,023 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:11:13,024 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:11:15,025 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:11:15,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:11:15,364 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:11:17,026 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:11:20,028 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:11:22,029 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:11:23,832 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 20:11:24,029 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:11:26,030 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:11:28,031 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:11:30,032 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:11:30,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:11:30,590 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:11:32,033 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:11:34,034 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:11:36,035 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:11:38,036 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:11:40,037 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:11:42,038 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:11:44,039 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:11:45,937 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:11:45,938 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:11:46,040 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:11:48,040 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:11:50,041 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:11:52,042 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:11:53,909 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 20:11:54,043 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:11:56,044 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:11:58,045 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:12:00,046 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:12:01,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:12:01,750 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:12:02,047 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:12:04,048 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:12:06,049 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:12:08,050 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:12:10,051 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:12:12,052 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:12:14,052 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:12:16,053 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:12:17,086 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:12:17,086 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:12:18,054 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:12:20,055 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:12:22,056 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:12:23,980 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 20:12:24,057 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:12:26,058 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:12:28,059 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:12:30,060 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:12:32,060 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:12:32,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:12:32,347 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:12:34,061 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:12:36,062 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:12:38,063 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:12:40,064 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:12:42,065 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:12:44,066 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:12:47,067 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:12:47,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:12:47,569 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:12:49,068 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:12:51,069 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:12:53,070 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:12:54,043 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 20:12:55,070 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:12:57,071 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:12:59,072 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:13:01,073 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:13:02,872 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:13:02,872 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:13:03,074 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:13:05,075 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:13:07,076 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:13:09,076 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:13:11,077 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:13:13,078 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:13:15,079 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:13:17,080 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:13:18,095 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:13:18,095 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:13:19,080 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:13:21,081 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:13:23,082 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:13:24,116 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 20:13:25,083 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:13:27,084 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:13:29,084 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:13:31,085 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:13:33,086 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:13:33,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:13:33,354 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:13:35,086 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:13:37,087 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:13:39,088 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:13:41,089 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:13:43,090 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:13:45,090 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:13:47,091 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:13:48,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:13:48,573 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:13:50,093 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:13:52,093 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:13:54,094 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:13:54,192 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 20:13:56,095 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:13:58,096 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:14:00,097 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:14:02,098 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:14:03,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:14:03,895 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:14:04,098 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:14:06,099 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:14:08,101 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:14:10,101 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:14:12,102 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:14:14,103 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:14:16,104 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:14:18,105 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:14:19,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:14:19,109 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:14:20,106 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:14:22,107 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:14:24,108 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:14:24,267 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 20:14:26,109 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:14:28,110 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:14:30,111 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:14:32,111 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:14:34,112 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:14:34,336 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:14:34,337 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:14:36,113 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:14:38,114 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:14:40,115 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:14:42,116 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:14:44,117 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:14:46,118 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:14:48,119 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:14:49,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:14:49,655 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:14:50,119 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:14:52,120 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:14:54,121 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:14:54,337 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 20:14:56,122 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:14:58,123 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:15:00,123 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:15:02,125 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:15:04,125 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:15:04,875 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:15:04,875 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:15:06,126 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:15:08,127 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:15:11,128 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:15:13,130 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:15:15,131 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:15:17,132 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:15:19,133 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:15:20,141 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:15:20,141 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:15:21,134 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:15:23,135 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:15:24,408 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 20:15:25,136 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:15:27,137 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:15:29,138 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:15:31,139 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:15:33,140 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:15:35,141 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:15:35,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:15:35,505 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:15:37,142 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:15:39,143 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:15:41,144 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:15:43,144 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:15:45,145 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:15:47,146 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:15:49,147 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:15:50,803 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:15:50,804 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:15:51,148 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:15:53,149 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:15:54,481 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 20:15:55,150 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:15:57,151 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:15:59,152 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:16:01,153 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:16:03,153 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:16:05,154 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:16:06,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:16:06,109 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:16:07,155 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:16:09,156 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:16:11,157 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:16:13,158 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:16:15,159 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:16:17,160 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:16:19,161 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:16:21,162 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:16:21,331 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:16:21,331 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:16:23,163 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:16:24,545 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 20:16:25,164 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:16:27,165 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:16:29,166 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:16:31,167 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:16:33,168 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:16:35,168 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:16:36,627 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:16:36,627 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:16:37,169 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:16:39,170 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:16:41,171 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:16:43,172 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:16:45,173 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:16:47,174 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:16:49,175 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:16:51,176 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:16:51,918 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:16:51,919 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:16:54,177 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:16:54,622 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 20:16:56,177 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:16:58,178 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:17:00,179 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:17:02,180 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:17:04,181 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:17:06,182 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:17:07,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:17:07,143 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:17:08,182 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:17:10,183 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:17:12,184 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:17:14,185 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:17:16,185 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:17:18,186 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:17:20,187 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:17:22,188 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:17:22,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:17:22,371 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:17:24,189 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:17:24,689 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 20:17:26,189 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:17:28,190 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:17:30,191 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:17:32,191 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:17:34,192 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:17:37,193 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:17:37,609 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:17:37,609 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:17:39,194 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:17:41,195 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:17:43,196 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:17:45,197 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:17:47,198 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:17:49,198 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:17:51,199 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:17:52,911 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:17:52,912 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:17:53,200 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:17:54,762 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 20:17:55,201 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:17:57,202 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:17:59,203 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:18:01,204 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:18:03,205 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:18:05,206 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:18:07,207 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:18:08,141 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:18:08,141 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:18:09,207 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:18:11,208 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:18:13,209 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:18:15,210 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:18:17,211 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:18:19,212 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:18:21,212 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:18:23,213 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:18:23,368 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:18:23,368 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:18:24,837 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 20:18:25,214 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:18:27,215 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:18:29,216 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:18:31,217 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:18:33,217 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:18:35,218 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:18:37,219 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:18:39,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:18:39,049 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:18:39,220 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:18:41,221 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:18:43,222 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:18:45,222 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:18:47,223 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:18:49,224 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:18:51,225 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:18:53,225 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:18:54,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:18:54,432 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:18:54,913 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 20:18:55,226 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:18:57,227 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:18:59,228 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:19:01,229 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:19:03,230 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:19:05,231 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:19:07,232 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:19:09,233 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:19:09,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:19:09,656 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:19:12,234 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:19:14,235 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:19:16,236 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:19:18,237 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:19:20,238 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:19:22,239 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:19:24,240 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:19:24,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:19:24,957 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:19:25,181 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 20:19:26,241 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:19:28,242 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:19:30,243 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:19:32,244 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:19:34,245 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:19:36,246 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:19:38,247 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:19:40,248 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:19:40,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:19:40,261 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:19:42,249 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:19:44,251 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:19:46,252 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:19:48,253 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:19:50,254 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:19:52,255 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:19:54,256 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:19:55,066 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 20:19:55,482 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:19:55,482 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:19:56,256 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:19:58,257 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:20:00,258 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:20:02,259 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:20:04,260 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:20:06,261 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:20:08,262 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:20:10,263 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:20:10,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:20:10,698 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:20:12,264 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:20:14,265 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:20:16,266 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:20:18,267 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:20:20,268 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:20:22,269 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:20:24,270 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:20:25,142 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 20:20:25,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:20:25,985 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:20:26,271 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:20:28,272 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:20:30,273 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:20:32,274 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:20:34,275 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:20:36,276 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:20:38,277 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:20:40,278 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:20:41,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:20:41,250 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:20:42,279 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:20:44,280 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:20:46,281 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:20:48,282 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:20:50,283 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:20:52,284 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:20:54,285 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:20:55,218 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 20:20:56,285 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:20:56,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:20:56,507 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:20:58,286 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:21:00,287 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:21:02,288 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:21:04,289 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:21:06,290 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:21:08,291 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:21:10,292 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:21:11,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:21:11,729 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:21:12,293 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:21:14,294 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:21:16,295 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:21:18,295 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:21:20,296 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:21:22,297 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:21:24,298 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:21:25,294 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 20:21:26,299 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:21:27,081 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:21:27,082 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:21:28,300 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:21:30,301 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:21:32,302 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:21:34,302 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:21:36,303 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:21:39,304 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:21:41,305 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:21:42,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:21:42,417 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:21:43,305 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:21:45,306 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:21:47,307 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:21:49,307 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:21:51,308 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:21:53,308 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:21:55,309 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:21:55,355 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 20:21:57,310 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:21:57,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:21:57,715 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:21:59,311 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:22:01,312 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:22:03,313 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:22:05,313 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:22:07,314 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:22:09,315 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:22:11,316 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:22:12,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:22:12,963 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:22:13,317 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:22:15,318 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:22:17,319 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:22:19,320 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:22:21,320 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:22:23,321 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:22:25,322 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:22:25,421 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 20:22:27,323 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:22:28,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:22:28,244 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:22:29,324 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:22:31,325 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:22:33,325 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:22:35,326 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:22:37,327 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:22:39,328 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:22:41,328 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:22:43,329 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:22:43,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:22:43,483 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:22:45,330 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:22:48,331 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:22:50,332 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:22:52,332 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:22:54,333 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:22:55,494 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 20:22:56,334 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:22:58,335 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:22:58,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:22:58,749 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:23:00,336 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:23:13,974 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:23:13,974 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:23:25,571 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 20:23:29,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:23:29,227 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:23:44,489 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:23:44,489 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:23:55,647 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 20:23:59,748 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:23:59,748 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:24:15,005 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:24:15,005 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:24:17,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history +2022-08-30 20:24:17,986 DEBUG SenderThread:3760704 [sender.py:send():232] send: history +2022-08-30 20:24:18,033 DEBUG SenderThread:3760704 [sender.py:send():232] send: summary +2022-08-30 20:24:18,098 INFO SenderThread:3760704 [sender.py:_save_file():946] saving file wandb-summary.json with policy end +2022-08-30 20:24:18,366 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/wandb-summary.json +2022-08-30 20:24:19,366 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:24:20,367 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:24:24,369 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:24:25,725 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 20:24:30,302 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:24:30,304 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:24:40,376 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log +2022-08-30 20:24:45,525 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:24:45,526 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:24:55,804 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 20:25:00,747 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:25:00,748 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:25:15,970 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:25:15,970 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:25:25,882 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 20:25:31,195 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:25:31,196 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:25:46,429 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:25:46,429 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:25:55,961 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 20:26:01,809 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:26:01,810 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:26:17,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:26:17,036 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:26:26,040 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 20:26:32,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:26:32,259 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:26:47,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:26:47,481 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:26:56,117 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 20:27:02,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:27:02,702 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:27:17,925 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:27:17,926 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:27:26,198 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 20:27:33,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:27:33,151 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:27:48,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:27:48,374 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:27:56,274 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +2022-08-30 20:28:03,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:28:03,598 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:28:18,827 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status +2022-08-30 20:28:18,828 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status +2022-08-30 20:28:26,350 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats