diff --git "a/wandb/run-20221025_092127-29xi74uq/logs/debug-internal.log" "b/wandb/run-20221025_092127-29xi74uq/logs/debug-internal.log" --- "a/wandb/run-20221025_092127-29xi74uq/logs/debug-internal.log" +++ "b/wandb/run-20221025_092127-29xi74uq/logs/debug-internal.log" @@ -57609,3 +57609,10929 @@ 2022-10-25 18:11:29,780 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status 2022-10-25 18:11:45,011 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status 2022-10-25 18:11:45,012 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:11:51,834 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:11:52,190 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:52,191 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:52,194 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:52,194 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:52,195 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:52,195 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:52,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:52,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:52,200 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:52,200 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:52,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:52,202 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:52,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:52,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:52,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:52,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:53,034 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:11:53,453 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:53,453 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:53,457 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:53,457 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:53,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:53,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:53,459 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:53,460 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:53,463 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:53,463 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:53,464 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:53,465 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:53,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:53,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:53,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:53,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:54,711 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:54,711 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:54,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:54,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:54,716 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:54,716 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:54,717 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:54,718 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:54,721 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:54,721 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:54,722 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:54,723 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:54,724 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:54,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:54,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:54,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:55,035 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:11:55,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:55,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:55,303 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:55,304 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:55,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:55,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:55,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:55,307 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:55,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:55,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:55,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:55,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:55,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:55,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:55,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:55,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:55,433 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:55,433 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:55,436 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:55,437 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:55,437 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:55,438 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:55,439 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:55,439 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:55,442 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:55,443 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:55,444 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:55,444 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:55,445 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:55,446 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:55,447 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:55,447 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:57,608 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:57,609 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:57,611 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:57,612 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:57,612 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:57,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:57,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:57,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:57,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:57,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:57,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:57,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:57,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:57,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:57,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:57,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:57,673 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:57,673 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:57,676 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:57,677 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:57,677 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:57,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:57,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:57,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:57,682 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:57,682 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:57,684 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:57,684 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:57,686 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:57,686 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:57,687 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:57,687 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:57,729 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:57,729 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:57,733 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:57,733 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:57,734 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:57,734 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:57,735 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:57,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:57,738 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:57,739 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:57,740 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:57,740 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:57,742 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:57,742 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:57,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:57,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:58,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:58,556 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:58,560 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:58,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:58,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:58,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:58,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:58,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:58,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:58,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:58,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:58,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:58,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:58,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:58,582 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:58,583 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:11:59,037 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:12:00,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:00,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:00,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:00,022 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:00,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:00,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:00,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:00,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:00,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:00,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:00,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:00,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:00,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:00,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:00,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:00,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:00,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:12:00,239 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:12:00,965 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:00,965 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:00,969 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:00,969 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:00,970 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:00,970 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:00,972 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:00,972 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:00,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:00,976 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:00,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:00,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:00,980 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:00,981 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:00,981 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:00,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:01,042 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:12:02,145 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:02,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:02,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:02,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:02,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:02,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:02,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:02,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:02,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:02,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:02,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:02,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:02,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:02,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:02,159 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:02,159 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:03,043 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:12:03,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:03,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:03,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:03,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:03,976 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:03,976 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:03,976 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:03,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:03,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:03,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:03,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:03,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:03,979 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:03,980 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:03,980 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:03,981 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:05,043 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:12:06,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:06,840 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:06,843 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:06,843 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:06,844 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:06,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:06,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:06,847 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:06,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:06,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:06,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:06,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:06,853 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:06,853 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:06,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:06,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:07,044 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:12:07,807 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:07,807 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:07,810 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:07,810 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:07,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:07,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:07,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:07,813 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:07,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:07,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:07,816 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:07,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:07,818 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:07,819 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:07,819 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:07,820 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:07,856 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:07,857 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:07,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:07,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:07,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:07,861 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:07,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:07,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:07,865 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:07,865 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:07,866 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:07,867 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:07,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:07,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:07,869 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:07,869 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:07,890 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:07,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:07,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:07,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:07,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:07,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:07,896 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:07,896 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:07,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:07,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:07,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:07,901 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:07,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:07,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:07,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:07,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:08,800 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:08,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:08,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:08,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:08,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:08,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:08,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:08,807 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:08,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:08,810 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:08,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:08,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:08,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:08,813 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:08,813 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:08,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:08,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:08,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:08,898 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:08,898 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:08,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:08,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:08,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:08,901 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:08,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:08,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:08,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:08,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:08,907 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:08,907 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:08,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:08,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:09,045 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:12:11,046 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:12:11,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:11,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:11,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:11,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:11,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:11,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:11,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:11,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:11,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:11,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:11,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:11,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:11,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:11,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:11,582 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:11,582 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:12,493 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:12,494 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:12,497 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:12,497 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:12,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:12,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:12,499 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:12,500 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:12,502 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:12,502 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:12,504 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:12,504 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:12,505 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:12,506 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:12,506 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:12,507 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:13,047 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:12:15,049 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:12:15,539 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:12:15,540 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:12:15,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:15,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:15,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:15,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:15,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:15,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:15,582 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:15,583 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:15,585 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:15,586 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:15,586 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:15,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:15,588 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:15,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:15,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:15,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:17,050 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:12:17,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:17,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:17,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:17,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:17,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:17,145 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:17,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:17,147 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:17,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:17,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:17,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:17,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:17,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:17,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:17,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:17,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:19,050 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:12:20,190 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:20,191 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:20,193 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:20,194 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:20,194 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:20,195 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:20,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:20,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:20,198 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:20,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:20,200 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:20,200 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:20,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:20,202 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:20,202 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:20,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:21,052 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:12:21,910 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:12:22,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:22,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:22,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:22,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:22,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:22,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:22,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:22,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:22,907 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:22,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:22,909 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:22,909 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:22,910 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:22,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:22,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:22,912 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:23,052 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:12:25,053 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:12:27,054 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:12:29,055 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:12:30,941 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:12:30,941 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:12:31,056 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:12:33,056 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:12:35,057 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:12:37,058 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:12:39,059 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:12:43,061 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:12:46,192 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:12:46,193 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:12:48,063 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:12:50,064 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:12:51,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:51,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:51,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:51,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:51,909 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:51,909 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:51,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:51,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:51,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:51,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:51,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:51,919 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:51,920 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:51,921 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:51,921 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:51,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:51,990 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:12:53,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:53,309 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:53,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:53,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:53,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:53,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:53,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:53,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:53,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:53,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:53,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:53,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:53,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:53,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:53,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:53,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:54,066 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:12:58,067 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:12:58,364 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:58,364 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:58,368 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:58,368 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:58,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:58,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:58,371 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:58,371 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:58,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:58,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:58,378 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:58,378 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:58,380 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:58,380 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:58,381 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:12:58,382 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:01,423 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:13:01,423 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:13:02,069 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:13:04,070 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:13:08,072 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:13:10,073 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:13:12,074 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:13:15,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:15,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:15,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:15,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:15,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:15,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:15,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:15,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:15,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:15,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:15,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:15,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:15,047 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:15,047 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:15,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:15,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:16,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:16,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:16,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:16,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:16,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:16,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:16,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:16,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:16,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:16,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:16,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:16,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:16,063 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:16,063 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:16,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:16,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:16,072 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:16,073 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:16,075 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:13:16,076 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:16,076 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:16,077 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:16,078 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:16,079 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:16,079 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:16,083 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:16,083 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:16,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:16,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:16,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:16,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:16,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:16,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:16,516 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:16,516 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:16,520 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:16,520 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:16,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:16,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:16,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:16,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:16,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:16,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:16,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:16,530 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:16,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:16,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:16,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:16,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:16,658 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:13:16,658 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:13:18,076 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:13:22,066 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:13:22,078 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:13:24,079 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:13:24,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:24,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:24,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:24,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:24,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:24,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:24,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:24,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:24,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:24,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:24,157 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:24,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:24,159 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:24,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:24,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:24,161 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:25,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:25,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:25,211 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:25,211 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:25,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:25,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:25,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:25,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:25,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:25,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:25,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:25,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:25,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:25,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:25,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:25,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:26,079 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:13:28,080 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:13:28,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:28,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:28,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:28,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:28,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:28,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:28,119 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:28,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:28,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:28,124 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:28,125 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:28,126 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:28,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:28,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:28,129 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:28,129 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:29,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:29,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:29,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:29,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:29,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:29,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:29,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:29,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:29,046 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:29,047 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:29,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:29,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:29,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:29,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:29,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:29,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:30,398 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:30,399 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:30,402 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:30,403 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:30,404 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:30,404 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:30,405 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:30,406 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:30,409 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:30,409 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:30,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:30,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:30,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:30,413 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:30,414 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:30,414 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:31,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:31,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:31,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:31,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:31,309 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:31,309 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:31,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:31,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:31,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:31,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:31,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:31,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:31,317 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:31,318 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:31,318 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:31,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:32,070 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:13:32,070 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:13:32,082 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:13:34,083 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:13:35,081 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:35,081 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:35,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:35,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:35,086 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:35,086 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:35,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:35,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:35,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:35,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:35,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:35,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:35,094 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:35,094 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:35,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:35,096 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:35,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:35,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:35,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:35,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:35,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:35,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:35,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:35,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:35,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:35,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:35,114 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:35,114 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:35,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:35,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:35,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:35,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:35,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:35,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:35,137 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:35,137 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:35,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:35,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:35,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:35,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:35,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:35,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:35,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:35,145 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:35,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:35,147 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:35,147 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:35,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:35,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:35,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:35,779 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:35,781 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:36,084 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:13:36,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:36,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:36,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:36,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:36,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:36,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:36,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:36,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:37,192 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:37,193 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:37,195 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:37,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:37,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:37,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:37,198 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:37,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:37,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:37,202 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:37,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:37,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:37,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:37,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:37,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:37,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:37,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:37,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:37,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:37,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:37,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:37,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:37,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:37,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:37,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:37,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:37,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:37,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:37,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:37,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:37,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:37,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:38,085 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:13:38,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:38,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:38,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:38,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:38,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:38,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:38,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:38,124 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:38,126 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:38,127 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:38,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:38,129 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:38,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:38,131 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:38,131 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:38,132 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:39,456 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:39,456 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:39,459 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:39,460 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:39,460 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:39,461 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:39,462 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:39,462 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:39,465 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:39,465 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:39,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:39,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:39,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:39,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:39,470 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:39,470 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:39,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:39,593 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:39,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:39,597 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:39,597 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:39,598 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:39,599 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:39,600 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:39,603 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:39,603 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:39,607 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:39,607 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:39,608 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:39,609 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:39,610 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:39,610 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:40,086 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:13:40,334 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:40,334 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:40,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:40,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:40,338 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:40,338 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:40,340 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:40,340 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:40,343 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:40,343 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:40,344 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:40,345 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:40,346 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:40,347 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:40,347 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:40,348 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:40,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:40,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:40,415 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:40,415 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:40,416 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:40,416 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:40,417 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:40,418 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:40,421 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:40,421 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:40,423 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:40,423 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:40,424 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:40,425 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:40,425 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:40,426 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:40,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:40,655 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:40,658 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:40,659 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:40,659 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:40,660 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:40,661 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:40,661 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:40,664 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:40,665 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:40,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:40,667 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:40,668 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:40,669 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:40,669 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:40,670 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:41,504 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:41,505 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:41,508 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:41,508 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:41,509 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:41,509 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:41,510 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:41,511 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:41,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:41,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:41,515 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:41,516 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:41,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:41,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:41,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:41,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:41,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:41,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:41,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:41,547 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:41,547 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:41,548 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:41,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:41,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:41,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:41,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:41,554 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:41,555 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:41,556 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:41,556 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:41,557 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:41,558 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:42,087 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:13:43,563 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:43,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:43,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:43,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:43,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:43,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:43,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:43,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:43,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:43,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:43,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:43,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:43,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:43,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:43,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:43,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:44,088 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:13:44,657 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:44,658 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:44,665 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:44,665 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:44,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:44,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:44,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:44,667 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:44,667 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:44,667 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:44,668 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:44,669 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:44,670 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:44,670 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:44,671 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:44,672 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:44,744 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:44,745 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:44,748 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:44,748 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:44,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:44,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:44,750 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:44,751 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:44,753 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:44,754 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:44,755 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:44,755 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:44,756 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:44,757 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:44,758 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:44,758 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:45,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:45,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:45,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:45,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:45,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:45,626 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:45,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:45,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:45,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:45,632 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:45,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:45,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:45,636 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:45,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:45,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:45,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:46,089 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:13:46,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:46,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:46,898 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:46,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:46,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:46,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:46,901 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:46,901 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:46,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:46,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:46,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:46,906 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:46,907 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:46,907 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:46,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:46,909 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:47,372 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:13:47,372 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:13:47,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:47,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:47,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:47,909 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:47,909 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:47,910 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:47,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:47,912 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:47,914 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:47,914 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:47,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:47,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:47,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:47,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:47,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:47,919 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:47,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:47,959 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:47,962 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:47,962 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:47,963 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:47,963 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:47,964 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:47,965 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:47,967 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:47,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:47,969 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:47,970 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:47,971 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:47,971 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:47,972 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:47,972 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:48,090 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:13:49,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:49,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:49,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:49,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:49,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:49,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:49,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:49,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:49,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:49,647 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:49,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:49,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:49,650 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:49,650 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:49,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:49,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:50,091 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:13:52,091 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:13:52,146 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:13:53,246 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:53,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:53,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:53,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:53,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:53,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:53,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:53,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:53,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:53,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:53,257 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:53,258 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:53,259 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:53,259 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:53,260 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:53,260 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:53,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:53,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:53,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:53,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:53,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:53,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:53,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:53,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:53,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:53,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:53,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:53,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:53,331 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:53,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:53,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:53,333 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:54,092 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:13:54,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:54,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:54,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:54,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:54,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:54,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:54,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:54,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:54,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:54,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:54,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:54,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:54,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:54,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:54,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:54,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:54,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:54,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:54,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:54,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:54,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:54,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:54,292 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:54,292 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:54,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:54,295 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:54,296 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:54,296 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:54,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:54,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:54,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:54,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:55,246 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:55,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:55,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:55,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:55,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:55,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:55,252 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:55,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:55,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:55,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:55,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:55,257 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:55,258 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:55,259 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:55,259 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:55,260 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:56,093 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:13:56,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:56,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:56,985 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:56,986 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:56,986 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:56,987 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:56,988 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:56,988 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:56,990 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:56,991 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:56,992 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:56,992 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:56,994 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:56,994 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:56,995 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:56,995 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:58,094 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:13:58,670 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:58,671 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:58,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:58,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:58,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:58,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:58,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:58,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:58,680 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:58,680 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:58,680 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:58,681 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:58,682 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:58,682 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:58,683 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:58,683 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:59,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:59,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:59,340 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:59,340 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:59,341 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:59,341 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:59,342 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:59,343 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:59,345 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:59,345 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:59,346 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:59,347 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:59,348 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:59,348 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:59,349 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:59,349 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:59,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:59,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:59,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:59,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:59,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:59,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:59,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:59,530 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:59,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:59,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:59,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:59,534 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:59,535 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:59,536 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:59,536 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:13:59,537 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:00,095 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:14:01,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:01,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:01,096 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:01,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:01,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:01,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:01,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:01,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:01,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:01,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:01,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:01,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:01,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:01,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:01,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:01,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:02,096 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:14:02,434 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:02,434 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:02,437 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:02,437 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:02,438 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:02,438 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:02,439 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:02,440 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:02,442 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:02,442 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:02,443 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:02,444 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:02,445 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:02,446 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:02,446 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:02,447 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:02,597 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:14:02,597 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:14:04,015 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:04,016 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:04,018 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:04,019 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:04,019 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:04,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:04,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:04,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:04,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:04,024 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:04,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:04,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:04,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:04,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:04,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:04,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:04,097 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:14:06,097 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:14:06,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:06,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:06,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:06,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:06,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:06,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:06,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:06,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:06,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:06,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:06,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:06,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:06,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:06,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:06,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:06,331 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:08,098 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:14:08,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:08,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:08,303 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:08,304 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:08,304 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:08,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:08,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:08,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:08,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:08,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:08,309 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:08,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:08,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:08,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:08,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:08,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:10,099 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:14:12,100 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:14:14,101 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:14:16,102 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:14:17,837 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:14:17,838 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:14:18,103 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:14:20,103 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:14:22,104 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:14:22,226 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:14:28,106 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:14:32,108 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:14:33,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:14:33,056 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:14:37,110 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:14:39,111 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:14:43,113 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:14:43,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:43,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:43,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:43,855 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:43,855 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:43,856 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:43,857 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:43,858 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:43,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:43,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:43,864 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:43,865 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:43,866 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:43,867 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:43,867 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:43,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:45,125 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:45,126 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:45,129 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:45,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:45,131 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:45,131 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:45,132 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:45,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:45,137 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:45,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:45,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:45,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:45,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:45,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:45,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:45,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:47,115 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:14:48,262 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:48,262 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:48,266 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:48,266 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:48,267 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:48,267 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:48,269 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:48,269 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:48,273 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:48,273 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:48,275 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:48,276 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:48,277 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:48,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:48,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:48,279 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:14:48,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:14:48,290 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:14:49,115 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:14:52,307 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:14:53,117 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:14:55,118 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:14:57,119 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:15:01,121 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:15:03,122 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:15:03,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:15:03,638 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:15:05,123 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:15:05,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:05,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:05,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:05,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:05,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:05,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:05,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:05,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:05,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:05,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:05,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:05,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:05,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:05,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:05,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:05,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:06,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:06,157 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:06,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:06,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:06,161 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:06,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:06,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:06,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:06,167 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:06,167 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:06,168 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:06,169 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:06,171 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:06,171 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:06,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:06,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:07,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:07,912 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:07,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:07,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:07,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:07,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:07,919 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:07,919 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:07,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:07,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:07,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:07,925 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:07,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:07,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:07,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:07,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:08,702 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:08,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:08,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:08,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:08,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:08,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:08,709 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:08,709 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:08,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:08,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:08,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:08,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:08,717 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:08,717 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:08,718 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:08,718 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:09,124 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:15:11,125 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:15:12,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:12,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:12,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:12,783 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:12,783 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:12,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:12,785 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:12,785 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:12,789 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:12,789 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:12,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:12,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:12,793 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:12,793 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:12,794 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:12,794 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:13,126 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:15:14,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:14,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:14,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:14,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:14,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:14,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:14,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:14,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:14,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:14,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:14,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:14,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:14,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:14,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:14,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:14,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:15,127 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:15:17,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:17,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:17,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:17,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:17,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:17,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:17,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:17,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:17,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:17,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:17,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:17,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:17,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:17,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:17,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:17,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:18,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:18,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:18,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:18,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:18,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:18,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:18,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:18,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:18,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:18,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:18,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:18,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:18,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:18,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:18,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:18,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:18,864 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:15:18,864 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:15:19,129 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:15:19,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:19,636 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:19,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:19,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:19,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:19,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:19,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:19,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:19,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:19,645 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:19,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:19,647 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:19,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:19,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:19,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:19,650 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:19,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:19,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:19,729 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:19,729 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:19,730 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:19,730 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:19,731 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:19,732 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:19,735 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:19,735 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:19,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:19,737 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:19,738 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:19,739 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:19,739 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:19,740 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:19,876 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:19,876 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:19,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:19,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:19,881 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:19,881 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:19,882 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:19,883 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:19,886 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:19,886 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:19,888 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:19,888 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:19,889 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:19,890 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:19,890 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:19,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:20,535 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:20,536 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:20,536 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:20,537 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:21,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:21,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:21,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:21,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:21,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:21,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:21,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:21,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:21,130 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:15:21,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:21,304 DEBUG SenderThread:2194005 [sender.py:send():232] send: history +2022-10-25 18:15:21,520 DEBUG SenderThread:2194005 [sender.py:send():232] send: summary +2022-10-25 18:15:21,589 INFO SenderThread:2194005 [sender.py:_save_file():946] saving file wandb-summary.json with policy end +2022-10-25 18:15:22,130 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/wandb-summary.json +2022-10-25 18:15:22,176 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:22,176 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:22,179 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:22,179 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:22,180 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:22,180 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:22,182 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:22,182 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:22,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:22,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:22,187 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:22,187 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:22,188 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:22,189 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:22,190 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:22,190 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:22,398 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:15:22,999 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:22,999 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:23,002 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:23,004 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:23,005 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:23,006 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:23,008 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:23,009 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:23,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:23,014 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:23,016 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:23,018 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:23,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:23,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:23,022 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:23,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:23,131 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:15:24,131 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:15:25,132 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:15:25,193 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:25,194 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:25,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:25,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:25,198 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:25,198 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:25,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:25,200 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:25,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:25,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:25,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:25,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:25,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:25,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:25,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:25,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:26,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:26,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:26,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:26,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:26,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:26,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:26,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:26,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:26,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:26,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:26,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:26,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:26,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:26,244 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:26,244 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:26,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:26,269 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:26,270 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:26,273 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:26,273 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:26,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:26,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:26,275 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:26,276 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:26,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:26,279 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:26,280 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:26,281 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:26,282 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:26,282 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:26,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:26,284 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:27,133 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:15:28,430 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,430 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,434 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,434 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,435 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,435 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,436 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,437 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,440 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,440 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,442 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,442 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,443 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,444 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,445 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,445 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,647 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,647 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,650 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,657 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,658 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,659 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,660 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,660 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,661 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,662 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,714 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,718 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,718 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,719 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,720 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,721 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,721 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,724 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,724 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,728 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,728 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,729 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,729 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,740 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,740 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,740 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,741 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,742 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,745 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,746 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,747 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,748 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,750 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:28,750 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:29,133 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:15:29,536 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:29,537 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:29,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:29,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:29,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:29,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:29,542 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:29,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:29,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:29,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:29,547 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:29,548 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:29,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:29,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:29,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:29,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:30,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:30,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:30,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:30,544 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:30,544 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:30,545 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:30,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:30,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:30,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:30,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:30,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:30,551 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:30,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:30,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:30,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:30,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:30,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:30,593 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:30,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:30,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:30,597 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:30,597 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:30,598 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:30,599 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:30,601 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:30,602 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:30,603 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:30,603 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:30,604 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:30,605 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:30,606 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:30,606 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:31,134 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:15:31,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:31,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:31,483 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:31,483 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:31,484 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:31,485 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:31,486 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:31,486 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:31,489 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:31,489 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:31,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:31,491 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:31,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:31,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:31,493 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:31,494 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:33,135 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:15:34,202 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:15:34,203 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:15:35,136 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:15:35,281 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:35,282 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:35,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:35,287 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:35,287 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:35,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:35,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:35,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:35,292 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:35,293 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:35,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:35,295 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:35,296 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:35,297 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:35,297 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:35,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:35,494 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:35,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:35,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:35,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:35,499 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:35,500 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:35,501 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:35,501 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:35,503 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:35,504 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:35,505 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:35,506 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:35,507 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:35,507 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:35,508 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:35,508 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:36,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:36,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:36,317 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:36,318 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:36,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:36,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:36,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:36,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:36,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:36,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:36,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:36,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:36,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:36,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:36,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:36,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:37,137 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:15:37,408 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:37,408 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:37,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:37,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:37,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:37,413 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:37,414 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:37,414 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:37,417 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:37,417 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:37,418 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:37,419 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:37,420 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:37,421 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:37,421 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:37,422 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:39,016 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:39,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:39,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:39,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:39,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:39,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:39,022 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:39,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:39,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:39,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:39,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:39,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:39,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:39,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:39,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:39,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:39,138 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:15:39,192 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:39,192 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:39,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:39,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:39,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:39,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:39,198 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:39,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:39,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:39,202 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:39,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:39,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:39,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:39,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:39,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:39,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:39,271 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:39,272 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:39,275 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:39,275 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:39,276 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:39,276 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:39,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:39,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:39,280 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:39,281 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:39,282 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:39,282 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:39,284 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:39,284 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:39,285 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:39,285 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:40,836 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:40,837 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:40,840 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:40,840 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:40,841 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:40,841 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:40,842 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:40,843 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:40,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:40,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:40,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:40,847 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:40,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:40,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:40,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:40,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:41,139 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:15:42,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:42,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:42,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:42,582 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:42,582 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:42,583 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:42,584 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:42,584 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:42,586 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:42,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:42,588 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:42,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:42,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:42,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:42,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:42,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:43,140 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:15:43,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:43,333 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:43,336 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:43,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:43,338 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:43,338 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:43,339 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:43,340 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:43,342 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:43,343 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:43,344 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:43,344 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:43,345 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:43,346 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:43,346 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:43,347 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:43,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:43,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:43,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:43,544 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:43,544 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:43,545 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:43,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:43,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:43,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:43,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:43,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:43,551 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:43,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:43,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:43,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:43,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:44,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:44,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:44,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:44,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:44,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:44,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:44,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:44,244 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:44,244 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:44,244 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:44,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:44,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:44,246 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:44,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:44,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:44,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:45,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:45,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:45,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:45,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:45,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:45,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:45,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:45,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:45,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:45,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:45,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:45,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:45,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:45,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:45,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:45,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:45,140 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:15:46,824 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:46,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:46,828 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:46,828 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:46,829 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:46,830 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:46,831 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:46,831 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:46,833 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:46,833 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:46,835 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:46,835 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:46,836 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:46,837 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:46,837 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:46,838 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:47,141 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:15:48,211 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:48,211 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:48,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:48,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:48,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:48,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:48,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:48,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:48,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:48,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:48,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:48,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:48,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:48,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:48,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:48,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:48,297 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:48,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:48,301 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:48,301 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:48,302 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:48,302 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:48,303 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:48,304 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:48,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:48,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:48,307 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:48,309 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:48,309 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:48,309 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:48,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:48,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:49,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:49,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:49,063 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:49,063 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:49,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:49,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:49,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:49,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:49,068 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:49,068 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:49,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:49,070 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:49,071 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:49,072 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:49,072 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:49,073 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:49,142 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:15:49,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:15:49,493 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:15:49,704 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:49,704 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:49,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:49,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:49,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:49,709 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:49,710 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:49,710 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:49,712 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:49,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:49,714 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:49,714 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:49,716 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:49,716 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:49,717 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:49,717 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:51,143 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:15:51,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:51,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:51,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:51,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:51,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:51,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:51,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:51,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:51,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:51,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:51,244 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:51,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:51,246 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:51,246 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:51,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:51,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:52,475 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:15:52,611 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:52,611 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:52,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:52,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:52,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:52,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:52,616 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:52,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:52,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:52,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:52,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:52,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:52,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:52,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:52,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:52,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:53,144 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:15:55,145 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:15:57,146 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:15:57,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:57,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:57,693 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:57,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:57,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:57,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:57,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:57,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:57,698 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:57,698 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:57,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:57,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:57,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:57,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:57,702 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:57,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:15:59,147 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:16:01,148 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:16:03,149 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:16:04,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:16:04,840 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:16:05,149 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:16:07,150 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:16:09,151 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:16:13,153 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:16:17,155 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:16:20,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:16:20,212 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:16:21,156 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:16:22,553 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:16:25,158 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:16:29,159 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:16:31,160 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:16:35,162 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:16:35,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:16:35,590 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:16:36,626 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:36,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:36,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:36,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:36,632 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:36,632 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:36,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:36,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:36,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:36,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:36,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:36,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:36,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:36,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:36,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:36,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:37,872 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:37,873 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:37,876 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:37,877 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:37,877 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:37,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:37,879 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:37,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:37,883 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:37,884 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:37,885 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:37,886 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:37,887 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:37,888 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:37,888 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:37,889 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:40,164 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:16:40,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:40,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:40,986 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:40,986 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:40,987 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:40,987 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:40,988 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:40,989 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:40,993 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:40,993 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:40,995 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:40,995 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:40,997 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:40,997 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:40,998 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:40,998 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:42,165 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:16:46,167 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:16:48,167 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:16:50,168 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:16:50,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:16:50,812 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:16:52,631 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:16:54,170 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:16:54,965 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:54,966 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:54,969 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:54,969 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:54,970 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:54,971 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:54,972 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:54,972 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:54,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:54,976 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:54,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:54,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:54,979 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:54,980 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:54,980 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:54,981 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:16:56,171 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:16:58,172 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:17:00,172 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:17:01,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:01,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:01,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:01,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:01,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:01,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:01,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:01,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:01,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:01,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:01,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:01,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:01,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:01,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:01,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:01,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:02,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:02,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:02,132 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:02,132 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:02,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:02,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:02,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:02,135 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:02,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:02,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:02,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:02,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:02,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:02,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:02,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:02,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:03,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:03,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:03,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:03,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:03,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:03,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:03,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:03,257 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:03,260 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:03,260 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:03,261 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:03,262 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:03,263 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:03,264 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:03,265 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:03,265 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:04,174 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:17:04,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:04,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:04,824 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:04,824 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:04,824 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:04,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:04,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:04,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:04,826 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:04,827 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:04,828 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:04,829 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:04,830 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:04,831 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:04,831 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:04,832 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:06,008 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:06,008 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:06,012 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:06,012 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:06,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:06,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:06,015 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:06,015 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:06,018 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:06,019 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:06,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:06,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:06,022 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:06,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:06,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:06,024 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:06,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:17:06,033 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:17:06,175 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:17:07,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:07,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:07,826 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:07,827 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:08,176 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:17:08,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:08,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:08,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:08,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:08,291 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:08,291 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:08,292 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:08,292 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:10,177 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:17:10,375 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:10,375 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:10,379 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:10,379 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:10,380 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:10,380 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:10,382 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:10,382 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:10,385 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:10,386 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:10,387 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:10,387 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:10,389 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:10,389 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:10,390 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:10,390 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:11,270 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:11,271 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:11,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:11,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:11,275 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:11,275 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:11,276 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:11,277 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:11,280 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:11,280 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:11,282 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:11,282 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:11,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:11,284 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:11,284 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:11,285 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:11,383 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:11,384 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:11,387 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:11,387 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:11,388 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:11,388 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:11,389 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:11,390 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:11,393 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:11,393 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:11,394 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:11,395 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:11,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:11,397 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:11,397 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:11,398 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:11,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:11,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:11,472 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:11,472 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:11,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:11,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:11,474 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:11,475 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:11,477 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:11,478 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:11,479 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:11,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:11,481 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:11,481 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:11,482 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:11,482 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:12,178 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:17:12,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:12,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:12,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:12,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:12,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:12,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:12,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:12,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:12,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:12,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:12,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:12,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:12,626 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:12,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:12,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:12,628 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:14,178 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:17:14,727 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:14,727 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:14,731 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:14,731 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:14,732 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:14,733 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:14,734 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:14,734 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:14,737 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:14,737 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:14,738 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:14,739 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:14,740 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:14,741 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:14,741 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:14,742 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:15,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:15,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:15,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:15,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:15,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:15,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:15,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:15,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:15,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:15,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:15,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:15,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:15,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:15,582 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:15,583 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:15,584 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:16,179 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:17:16,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:16,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:16,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:16,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:16,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:16,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:16,597 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:16,597 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:16,600 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:16,600 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:16,601 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:16,602 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:16,603 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:16,604 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:16,604 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:16,605 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:17,558 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:17,558 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:17,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:17,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:17,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:17,563 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:17,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:17,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:17,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:17,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:17,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:17,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:17,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:17,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:17,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:17,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:18,180 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:17:18,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:18,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:18,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:18,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:18,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:18,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:18,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:18,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:18,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:18,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:18,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:18,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:18,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:18,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:18,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:18,582 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:19,764 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:19,765 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:19,767 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:19,768 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:19,768 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:19,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:19,770 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:19,770 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:19,773 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:19,773 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:19,774 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:19,775 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:19,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:19,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:19,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:19,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:19,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:19,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:19,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:19,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:19,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:19,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:19,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:19,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:19,807 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:19,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:19,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:19,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:19,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:19,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:19,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:19,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,181 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:17:20,586 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,586 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,593 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,597 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,598 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,598 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,599 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,599 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,626 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,630 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,632 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,633 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,787 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,789 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,790 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,793 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,793 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,796 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,797 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,797 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,798 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,852 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,855 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,855 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,856 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,856 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,858 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,858 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,861 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,863 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,864 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,864 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,865 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:20,865 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:21,335 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:17:21,336 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:17:21,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:21,628 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:21,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:21,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:21,632 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:21,633 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:21,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:21,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:21,636 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:21,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:21,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:21,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:21,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:21,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:21,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:21,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:22,182 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:17:22,710 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:17:23,773 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:23,773 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:23,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:23,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:23,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:23,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:23,779 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:23,779 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:23,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:23,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:23,783 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:23,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:23,785 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:23,785 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:23,786 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:23,787 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:24,183 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:17:26,184 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:17:27,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:27,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:27,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:27,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:27,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:27,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:27,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:27,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:27,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:27,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:27,065 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:27,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:27,067 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:27,068 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:27,068 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:27,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:27,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:27,840 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:27,843 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:27,844 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:27,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:27,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:27,847 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:27,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:27,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:27,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:27,852 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:27,852 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:27,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:27,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:27,855 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:27,856 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:27,920 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:27,920 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:27,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:27,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:27,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:27,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:27,925 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:27,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:27,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:27,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:27,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:27,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:27,931 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:27,932 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:27,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:27,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:28,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:28,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:28,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:28,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:28,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:28,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:28,063 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:28,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:28,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:28,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:28,067 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:28,068 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:28,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:28,070 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:28,070 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:28,071 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:28,185 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:17:29,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:29,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:29,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:29,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:29,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:29,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:29,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:29,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:29,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:29,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:29,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:29,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:29,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:29,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:29,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:29,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:29,767 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:29,768 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:29,771 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:29,771 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:29,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:29,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:29,773 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:29,774 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:29,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:29,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:29,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:29,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:29,779 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:29,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:29,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:29,781 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:29,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:29,840 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:29,843 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:29,843 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:29,844 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:29,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:29,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:29,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:29,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:29,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:29,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:29,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:29,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:29,852 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:29,852 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:29,853 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:30,185 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:17:30,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:30,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:30,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:30,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:30,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:30,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:30,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:30,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:30,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:30,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:30,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:30,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:30,626 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:30,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:30,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:30,628 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:32,186 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:17:33,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:33,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:33,704 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:33,704 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:33,705 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:33,705 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:33,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:33,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:33,709 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:33,709 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:33,711 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:33,711 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:33,712 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:33,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:33,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:33,714 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:33,840 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:33,840 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:33,843 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:33,844 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:33,844 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:33,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:33,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:33,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:33,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:33,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:33,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:33,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:33,852 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:33,852 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:33,853 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:33,853 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:34,187 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:17:34,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:34,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:34,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:34,544 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:34,544 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:34,545 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:34,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:34,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:34,548 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:34,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:34,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:34,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:34,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:34,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:34,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:34,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:35,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:35,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:35,244 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:35,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:35,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:35,246 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:35,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:35,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:35,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:35,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:35,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:35,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:35,252 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:35,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:35,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:35,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:35,420 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:35,421 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:35,424 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:35,424 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:35,425 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:35,425 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:35,426 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:35,427 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:35,429 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:35,429 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:35,430 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:35,431 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:35,432 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:35,433 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:35,433 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:35,434 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:35,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:35,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:35,522 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:35,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:35,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:35,524 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:35,525 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:35,526 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:35,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:35,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:35,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:35,530 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:35,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:35,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:35,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:35,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:36,188 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:17:36,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:17:36,597 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:17:37,556 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:37,557 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:37,560 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:37,560 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:37,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:37,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:37,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:37,563 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:37,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:37,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:37,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:37,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:37,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:37,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:37,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:37,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:37,611 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:37,611 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:37,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:37,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:37,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:37,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:37,616 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:37,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:37,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:37,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:37,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:37,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:37,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:37,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:37,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:37,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:38,189 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:17:38,282 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:38,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:38,285 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:38,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:38,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:38,287 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:38,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:38,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:38,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:38,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:38,291 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:38,292 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:38,293 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:38,293 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:38,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:38,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:40,190 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:17:40,282 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:40,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:40,285 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:40,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:40,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:40,287 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:40,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:40,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:40,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:40,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:40,291 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:40,292 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:40,293 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:40,293 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:40,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:40,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:40,375 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:40,375 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:40,378 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:40,378 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:40,379 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:40,379 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:40,380 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:40,381 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:40,382 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:40,383 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:40,384 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:40,385 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:40,386 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:40,386 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:40,387 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:40,387 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:41,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:41,530 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:41,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:41,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:41,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:41,534 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:41,535 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:41,535 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:41,537 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:41,537 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:41,539 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:41,539 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:41,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:41,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:41,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:41,542 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:42,191 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:17:44,192 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:17:46,193 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:17:47,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:47,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:47,063 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:47,063 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:47,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:47,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:47,065 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:47,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:47,067 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:47,068 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:47,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:47,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:47,070 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:47,071 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:47,071 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:47,072 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:17:48,193 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:17:50,194 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:17:51,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:17:51,855 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:17:52,195 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:17:52,787 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:17:54,196 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:18:00,198 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:18:04,200 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:18:07,073 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:18:07,073 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:18:08,202 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:18:12,204 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:18:16,205 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:18:19,207 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:18:22,434 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:18:22,434 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:18:22,866 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:18:23,208 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:18:27,210 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:18:28,479 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:28,479 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:28,483 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:28,483 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:28,484 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:28,484 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:28,485 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:28,486 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:28,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:28,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:28,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:28,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:28,494 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:28,494 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:28,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:28,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:29,211 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:18:32,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:32,366 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:32,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:32,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:32,370 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:32,371 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:32,372 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:32,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:32,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:32,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:32,378 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:32,379 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:32,380 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:32,381 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:32,382 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:32,382 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:33,213 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:18:34,014 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:34,015 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:34,018 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:34,018 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:34,019 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:34,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:34,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:34,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:34,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:34,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:34,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:34,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:34,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:34,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:34,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:34,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:35,214 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:18:37,215 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:18:37,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:18:37,795 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:18:41,216 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:18:43,217 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:18:45,218 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:18:49,220 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:18:51,221 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:18:52,942 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:18:53,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:18:53,025 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:18:53,222 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:18:54,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:54,129 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:54,132 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:54,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:54,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:54,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:54,135 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:54,136 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:54,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:54,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:54,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:54,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:54,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:54,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:54,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:54,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:54,797 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:54,798 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:54,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:54,800 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:54,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:54,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:54,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:54,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:54,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:54,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:54,852 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:54,852 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:54,855 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:54,856 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:54,857 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:54,857 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:54,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:54,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:54,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:54,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:55,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:55,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:55,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:55,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:55,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:55,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:55,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:55,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:55,126 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:55,127 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:55,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:55,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:55,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:55,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:55,131 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:55,132 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:55,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:55,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:55,161 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:55,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:55,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:55,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:55,164 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:55,165 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:55,167 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:55,168 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:55,169 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:55,170 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:55,171 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:55,171 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:55,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:55,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:55,223 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:18:55,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:55,333 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:55,334 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:55,335 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:55,336 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:55,336 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:55,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:55,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:55,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:55,602 DEBUG SenderThread:2194005 [sender.py:send():232] send: history +2022-10-25 18:18:55,817 DEBUG SenderThread:2194005 [sender.py:send():232] send: summary +2022-10-25 18:18:55,890 INFO SenderThread:2194005 [sender.py:_save_file():946] saving file wandb-summary.json with policy end +2022-10-25 18:18:56,129 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:56,129 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:56,131 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:56,131 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:56,132 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:56,132 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:56,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:56,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:56,137 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:56,137 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:56,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:56,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:56,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:56,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:56,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:56,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:56,223 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:18:56,224 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/wandb-summary.json +2022-10-25 18:18:57,224 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:18:59,225 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:18:59,405 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:59,406 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:59,409 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:59,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:59,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:59,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:59,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:59,413 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:59,415 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:59,416 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:59,417 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:59,418 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:59,419 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:59,420 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:59,420 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:59,421 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:59,766 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:59,767 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:59,770 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:59,771 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:59,771 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:59,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:59,773 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:59,774 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:59,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:59,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:59,779 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:59,779 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:59,781 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:59,781 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:59,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:18:59,783 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:01,226 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:19:02,887 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:02,888 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:02,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:02,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:02,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:02,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:02,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:02,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:02,897 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:02,897 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:02,898 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:02,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:02,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:02,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:02,901 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:02,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:03,226 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:19:03,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:03,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:03,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:03,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:03,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:03,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:03,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:03,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:03,820 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:03,820 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:03,821 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:03,822 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:03,823 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:03,824 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:03,824 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:03,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:04,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:04,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:04,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:04,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:04,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:04,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:04,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:04,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:04,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:04,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:04,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:04,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:04,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:04,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:04,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:04,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:05,078 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:05,079 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:05,082 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:05,082 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:05,083 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:05,083 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:05,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:05,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:05,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:05,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:05,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:05,090 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:05,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:05,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:05,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:05,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:05,227 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:19:05,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:05,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:05,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:05,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:05,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:05,807 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:05,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:05,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:05,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:05,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:05,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:05,813 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:05,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:05,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:05,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:05,816 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:07,228 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:19:07,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:07,818 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:07,821 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:07,821 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:07,822 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:07,822 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:07,823 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:07,824 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:07,826 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:07,826 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:07,828 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:07,828 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:07,829 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:07,830 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:07,831 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:07,831 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:08,352 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:19:08,353 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:19:08,847 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:08,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:08,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:08,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:08,852 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:08,852 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:08,853 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:08,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:08,856 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:08,856 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:08,858 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:08,858 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:08,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:08,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:08,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:08,861 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:09,010 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:09,011 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:09,014 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:09,014 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:09,015 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:09,016 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:09,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:09,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:09,019 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:09,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:09,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:09,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:09,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:09,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:09,024 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:09,024 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:09,229 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:19:09,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:09,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:09,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:09,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:09,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:09,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:09,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:09,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:09,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:09,628 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:09,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:09,630 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:09,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:09,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:09,632 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:09,632 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:10,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:10,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:10,729 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:10,729 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:10,730 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:10,730 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:10,731 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:10,732 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:10,734 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:10,735 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:10,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:10,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:10,737 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:10,738 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:10,738 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:10,739 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:10,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:10,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:10,779 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:10,779 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:10,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:10,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:10,781 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:10,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:10,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:10,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:10,786 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:10,786 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:10,787 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:10,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:10,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:10,789 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:11,230 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:19:11,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:11,554 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:11,556 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:11,557 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:11,557 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:11,558 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:11,559 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:11,559 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:11,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:11,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:11,563 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:11,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:11,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:11,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:11,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:11,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:11,729 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:11,729 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:11,732 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:11,733 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:11,733 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:11,734 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:11,735 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:11,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:11,738 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:11,738 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:11,740 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:11,740 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:11,741 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:11,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:11,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:11,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:11,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:11,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:11,798 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:11,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:11,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:11,800 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:11,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:11,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:11,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:11,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:11,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:11,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:11,807 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:11,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:11,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:11,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:12,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:12,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:12,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:12,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:12,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:12,551 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:12,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:12,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:12,555 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:12,555 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:12,556 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:12,557 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:12,558 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:12,558 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:12,559 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:12,560 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:13,231 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:19:14,398 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:14,399 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:14,401 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:14,402 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:14,402 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:14,403 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:14,404 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:14,405 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:14,407 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:14,407 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:14,409 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:14,409 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:14,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:14,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:14,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:14,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:15,231 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:19:16,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:16,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:16,416 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:16,416 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:16,417 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:16,417 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:16,418 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:16,419 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:16,421 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:16,421 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:16,423 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:16,423 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:16,424 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:16,425 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:16,425 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:16,426 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:17,232 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:19:18,002 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:18,003 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:18,006 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:18,008 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:18,009 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:18,010 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:18,012 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:18,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:18,016 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:18,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:18,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:18,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:18,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:18,024 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:18,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:18,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:18,609 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:18,610 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:18,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:18,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:18,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:18,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:18,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:18,616 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:18,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:18,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:18,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:18,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:18,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:18,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:18,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:18,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:19,233 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:19:19,520 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:19,520 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:19,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:19,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:19,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:19,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:19,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:19,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:19,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:19,530 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:19,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:19,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:19,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:19,534 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:19,534 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:19,535 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:19,741 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:19,742 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:19,745 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:19,745 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:19,746 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:19,746 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:19,747 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:19,748 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:19,750 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:19,750 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:19,751 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:19,752 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:19,753 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:19,754 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:19,754 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:19,755 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:19,786 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:19,787 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:19,790 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:19,790 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:19,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:19,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:19,792 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:19,793 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:19,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:19,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:19,797 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:19,797 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:19,798 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:19,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:19,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:19,800 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:20,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:20,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:20,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:20,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:20,309 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:20,309 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:20,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:20,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:20,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:20,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:20,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:20,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:20,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:20,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:20,317 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:20,318 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:21,234 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:19:22,744 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:22,745 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:22,748 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:22,748 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:22,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:22,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:22,750 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:22,751 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:22,752 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:22,753 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:22,754 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:22,754 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:22,756 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:22,756 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:22,757 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:22,757 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:23,060 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:19:23,235 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:19:23,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:23,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:23,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:23,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:23,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:23,378 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:23,379 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:23,379 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:23,381 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:23,381 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:23,383 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:23,383 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:23,384 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:23,385 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:23,385 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:23,386 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:23,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:23,515 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:23,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:23,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:23,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:23,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:23,520 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:23,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:23,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:23,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:23,524 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:23,525 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:23,526 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:23,526 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:23,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:23,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:23,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:19:23,576 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:19:24,386 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:24,386 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:24,389 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:24,390 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:24,390 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:24,391 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:24,392 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:24,392 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:24,394 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:24,394 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:24,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:24,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:24,397 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:24,398 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:24,398 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:24,399 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:24,919 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:24,919 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:24,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:24,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:24,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:24,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:24,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:24,925 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:24,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:24,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:24,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:24,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:24,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:24,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:24,931 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:24,931 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:25,236 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:19:25,542 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:25,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:25,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:25,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:25,547 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:25,547 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:25,548 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:25,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:25,551 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:25,551 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:25,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:25,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:25,554 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:25,554 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:25,555 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:25,555 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:25,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:25,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:25,698 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:25,698 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:25,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:25,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:25,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:25,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:25,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:25,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:25,704 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:25,705 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:25,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:25,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:25,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:25,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:25,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:25,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:25,785 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:25,785 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:25,786 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:25,787 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:25,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:25,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:25,790 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:25,790 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:25,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:25,792 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:25,793 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:25,794 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:25,794 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:25,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:26,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:26,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:26,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:26,246 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:26,246 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:26,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:26,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:26,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:26,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:26,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:26,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:26,252 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:26,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:26,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:26,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:26,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:27,237 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:19:28,885 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:28,886 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:28,888 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:28,889 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:28,889 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:28,890 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:28,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:28,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:28,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:28,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:28,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:28,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:28,896 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:28,896 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:28,897 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:28,897 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:28,907 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:28,907 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:28,910 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:28,910 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:28,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:28,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:28,912 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:28,913 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:28,914 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:28,915 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:28,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:28,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:28,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:28,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:28,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:28,919 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:29,238 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:19:29,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:29,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:29,524 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:29,524 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:29,525 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:29,525 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:29,526 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:29,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:29,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:29,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:29,530 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:29,530 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:29,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:29,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:29,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:29,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:30,761 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:30,761 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:30,764 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:30,765 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:30,765 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:30,766 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:30,767 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:30,767 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:30,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:30,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:30,770 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:30,771 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:30,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:30,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:30,773 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:30,773 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:30,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:30,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:30,897 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:30,897 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:30,898 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:30,898 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:30,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:30,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:30,901 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:30,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:30,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:30,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:30,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:30,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:30,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:30,906 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:31,239 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:19:33,240 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:19:35,240 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:19:35,364 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:35,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:35,368 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:35,368 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:35,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:35,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:35,370 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:35,371 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:35,372 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:35,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:35,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:35,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:35,375 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:35,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:35,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:35,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:19:37,241 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:19:38,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:19:38,807 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:19:39,242 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:19:41,243 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:19:43,244 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:19:49,246 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:19:53,138 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:19:53,248 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:19:54,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:19:54,093 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:19:55,248 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:19:59,250 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:20:03,252 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:20:07,253 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:20:09,254 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:20:09,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:20:09,324 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:20:13,256 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:20:15,257 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:20:19,259 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:20:21,259 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:20:22,834 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:22,835 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:22,838 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:22,838 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:22,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:22,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:22,840 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:22,841 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:22,844 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:22,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:22,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:22,847 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:22,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:22,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:22,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:22,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:23,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:23,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:23,127 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:23,127 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:23,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:23,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:23,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:23,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:23,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:23,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:23,135 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:23,136 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:23,137 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:23,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:23,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:23,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:23,218 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:20:23,260 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:20:24,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:20:24,614 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:20:25,261 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:20:29,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:29,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:29,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:29,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:29,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:29,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:29,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:29,211 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:29,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:29,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:29,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:29,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:29,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:29,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:29,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:29,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:29,263 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:20:31,264 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:20:33,265 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:20:35,266 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:20:39,268 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:20:39,861 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:20:39,861 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:20:40,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:40,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:40,824 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:40,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:41,268 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:20:41,276 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:41,281 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:41,281 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:41,282 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:41,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:41,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:41,284 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:41,284 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:43,269 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:20:44,494 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:44,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:44,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:44,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:44,499 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:44,500 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:44,501 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:44,501 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:44,504 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:44,504 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:44,506 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:44,506 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:44,507 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:44,508 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:44,509 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:44,509 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:45,270 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:20:46,341 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:46,342 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:46,349 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:46,349 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:46,349 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:46,349 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:46,350 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:46,350 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:46,350 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:46,351 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:46,352 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:46,353 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:46,354 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:46,354 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:46,355 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:46,356 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:46,669 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:46,669 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:46,673 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:46,673 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:46,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:46,675 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:46,676 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:46,676 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:46,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:46,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:46,681 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:46,681 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:46,683 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:46,683 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:46,684 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:46,684 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:47,271 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:20:47,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:47,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:47,317 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:47,318 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:47,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:47,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:47,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:47,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:47,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:47,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:47,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:47,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:47,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:47,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:47,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:47,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:48,462 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:48,463 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:48,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:48,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:48,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:48,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:48,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:48,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:48,471 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:48,472 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:48,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:48,474 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:48,475 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:48,475 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:48,476 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:48,476 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:48,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:48,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:48,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:48,807 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:48,807 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:48,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:48,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:48,810 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:48,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:48,813 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:48,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:48,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:48,816 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:48,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:48,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:48,818 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:49,272 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:20:49,608 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:49,609 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:49,611 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:49,612 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:49,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:49,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:49,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:49,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:49,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:49,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:49,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:49,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:49,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:49,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:49,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:49,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:51,273 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:20:51,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:51,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:51,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:51,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:51,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:51,499 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:51,499 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:51,499 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:51,500 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:51,500 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:51,500 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:51,501 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:51,502 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:51,502 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:51,503 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:51,503 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:52,689 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:52,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:52,693 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:52,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:52,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:52,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:52,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:52,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:52,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:52,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:52,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:52,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:52,702 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:52,702 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:52,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:52,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:53,274 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:20:53,296 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:20:54,366 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:54,367 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:54,370 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:54,370 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:54,371 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:54,371 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:54,372 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:54,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:54,375 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:54,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:54,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:54,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:54,378 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:54,379 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:54,379 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:54,380 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:54,503 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:54,503 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:54,506 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:54,507 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:54,507 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:54,508 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:54,509 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:54,509 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:54,512 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:54,512 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:54,513 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:54,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:54,515 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:54,515 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:54,516 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:54,516 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:55,086 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:20:55,086 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:20:55,275 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:20:56,200 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:56,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:56,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:56,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:56,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:56,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:56,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:56,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:56,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:56,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:56,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:56,211 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:56,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:56,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:56,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:56,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:56,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:56,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:56,331 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:56,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:56,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:56,333 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:56,334 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:56,334 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:56,336 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:56,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:56,338 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:56,338 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:56,340 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:56,340 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:56,341 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:56,341 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:56,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:56,496 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:56,503 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:56,503 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:56,504 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:56,504 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:56,504 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:56,505 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:56,505 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:56,505 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:56,506 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:56,507 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:56,508 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:56,508 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:56,509 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:56,509 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:57,276 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:20:58,078 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:58,079 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:58,081 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:58,082 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:58,083 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:58,083 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:58,084 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:58,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:58,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:58,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:58,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:58,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:58,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:58,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:58,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:58,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:58,127 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:58,127 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:58,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:58,131 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:58,131 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:58,132 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:58,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:58,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:58,136 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:58,136 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:58,137 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:58,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:58,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:58,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:58,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:58,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:59,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:59,090 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:59,094 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:59,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:59,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:59,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:59,100 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:59,100 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:59,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:59,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:59,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:59,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:59,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:59,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:59,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:59,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:59,277 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:20:59,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:59,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:59,897 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:59,898 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:59,898 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:59,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:59,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:59,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:59,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:59,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:59,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:59,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:59,906 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:59,907 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:59,907 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:59,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:59,964 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:59,965 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:59,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:59,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:59,969 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:59,969 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:59,970 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:59,971 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:59,973 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:59,974 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:59,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:59,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:59,976 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:59,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:59,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:20:59,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:00,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:00,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:00,716 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:00,717 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:00,717 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:00,718 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:00,719 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:00,719 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:00,721 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:00,722 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:00,723 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:00,723 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:00,724 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:00,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:00,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:00,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:01,278 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:21:03,318 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:03,318 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:03,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:03,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:03,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:03,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:03,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:03,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:03,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:03,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:03,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:03,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:03,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:03,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:03,331 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:03,331 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,046 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,047 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,047 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,279 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:21:04,606 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,607 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,610 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,612 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,616 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,636 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,794 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,794 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,796 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,797 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,800 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,931 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,932 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,936 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,938 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:04,940 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:06,280 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:21:06,884 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:06,885 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:06,887 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:06,888 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:06,888 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:06,889 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:06,890 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:06,890 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:06,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:06,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:06,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:06,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:06,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:06,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:06,896 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:06,897 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:07,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:07,785 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:07,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:07,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:07,789 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:07,789 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:07,790 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:07,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:07,792 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:07,793 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:07,794 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:07,794 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:07,796 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:07,796 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:07,797 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:07,797 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:08,281 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:21:08,356 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:08,357 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:08,360 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:08,360 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:08,361 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:08,361 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:08,362 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:08,363 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:08,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:08,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:08,366 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:08,367 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:08,368 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:08,368 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:08,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:08,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:09,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:09,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:09,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:09,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:09,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:09,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:09,554 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:09,555 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:09,556 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:09,557 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:09,558 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:09,558 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:09,559 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:09,560 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:09,560 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:09,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:09,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:09,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:09,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:09,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:09,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:09,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:09,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:09,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:09,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:09,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:09,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:09,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:09,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:09,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:09,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:09,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:10,282 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:21:10,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:21:10,306 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:21:10,443 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:10,444 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:10,446 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:10,447 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:10,447 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:10,448 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:10,449 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:10,450 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:10,451 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:10,452 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:10,453 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:10,453 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:10,454 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:10,455 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:10,456 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:10,456 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:11,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:11,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:11,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:11,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:11,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:11,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:11,063 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:11,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:11,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:11,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:11,067 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:11,067 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:11,068 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:11,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:11,070 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:11,070 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:11,663 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:11,664 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:11,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:11,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:11,667 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:11,667 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:11,668 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:11,669 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:11,671 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:11,671 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:11,672 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:11,673 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:11,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:11,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:11,675 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:11,675 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:12,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:12,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:12,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:12,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:12,145 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:12,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:12,147 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:12,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:12,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:12,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:12,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:12,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:12,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:12,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:12,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:12,155 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:12,283 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:21:14,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,147 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,147 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,155 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,155 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,283 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:21:14,302 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,302 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,307 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:14,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:16,284 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:21:16,626 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:16,626 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:16,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:16,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:16,630 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:16,630 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:16,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:16,632 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:16,633 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:16,633 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:16,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:16,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:16,636 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:16,636 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:16,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:16,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:16,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:16,647 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:16,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:16,650 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:16,650 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:16,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:16,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:16,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:16,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:16,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:16,655 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:16,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:16,657 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:16,657 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:16,658 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:16,658 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:17,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:17,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:17,775 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:17,775 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:17,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:17,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:17,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:17,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:17,779 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:17,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:17,781 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:17,781 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:17,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:17,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:17,785 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:17,785 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:18,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:18,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:18,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:18,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:18,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:18,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:18,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:18,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:18,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:18,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:18,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:18,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:18,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:18,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:18,244 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:18,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:18,285 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:21:20,018 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:20,019 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:20,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:20,022 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:20,022 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:20,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:20,024 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:20,024 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:20,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:20,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:20,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:20,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:20,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:20,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:20,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:20,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:20,286 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:21:22,286 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:21:22,949 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:22,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:22,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:22,953 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:22,953 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:22,954 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:22,954 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:22,955 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:22,956 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:22,957 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:22,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:22,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:22,959 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:22,960 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:22,960 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:22,961 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:21:23,380 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:21:24,287 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:21:25,746 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:21:25,747 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:21:26,288 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:21:32,291 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:21:36,292 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:21:40,294 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:21:41,176 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:21:41,176 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:21:44,296 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:21:48,298 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:21:52,300 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:21:53,460 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:21:54,301 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:21:56,401 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:21:56,402 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:21:58,303 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:22:00,304 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:22:02,305 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:22:06,306 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:22:08,307 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:22:10,308 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:22:10,984 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:10,985 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:10,988 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:10,988 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:10,989 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:10,989 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:10,991 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:10,991 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:10,994 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:10,995 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:10,996 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:10,997 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:10,998 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:10,999 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:10,999 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:11,000 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:11,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:11,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:11,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:11,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:11,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:11,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:11,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:11,317 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:11,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:11,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:11,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:11,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:11,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:11,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:11,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:11,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:11,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:22:11,625 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:22:14,310 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:22:16,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:16,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:16,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:16,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:16,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:16,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:16,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:16,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:16,114 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:16,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:16,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:16,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:16,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:16,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:16,119 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:16,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:16,311 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:22:18,312 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:22:20,313 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:22:22,314 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:22:23,540 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:22:25,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:25,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:25,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:25,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:25,628 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:25,633 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:25,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:25,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:25,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:25,636 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:25,636 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:25,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:25,816 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:25,907 DEBUG SenderThread:2194005 [sender.py:send():232] send: history +2022-10-25 18:22:26,120 DEBUG SenderThread:2194005 [sender.py:send():232] send: summary +2022-10-25 18:22:26,188 INFO SenderThread:2194005 [sender.py:_save_file():946] saving file wandb-summary.json with policy end +2022-10-25 18:22:26,316 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:22:26,316 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/wandb-summary.json +2022-10-25 18:22:27,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:22:27,014 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:22:27,316 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:22:28,317 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:22:30,318 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:22:32,319 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:22:34,319 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:22:35,006 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:35,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:35,010 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:35,010 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:35,011 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:35,011 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:35,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:35,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:35,015 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:35,016 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:35,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:35,018 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:35,019 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:35,019 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:35,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:35,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:35,609 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:35,609 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:35,612 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:35,612 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:35,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:35,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:35,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:35,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:35,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:35,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:35,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:35,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:35,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:35,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:35,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:35,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:35,836 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:35,837 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:35,840 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:35,840 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:35,841 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:35,842 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:35,843 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:35,843 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:35,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:35,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:35,847 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:35,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:35,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:35,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:35,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:35,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:36,320 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:22:37,504 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:37,504 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:37,507 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:37,508 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:37,508 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:37,509 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:37,510 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:37,510 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:37,513 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:37,513 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:37,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:37,515 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:37,516 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:37,516 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:37,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:37,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:37,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:37,869 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:37,872 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:37,872 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:37,873 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:37,874 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:37,875 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:37,875 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:37,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:37,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:37,879 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:37,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:37,881 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:37,882 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:37,883 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:37,883 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:38,321 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:22:39,606 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:39,607 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:39,610 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:39,610 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:39,611 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:39,611 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:39,612 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:39,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:39,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:39,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:39,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:39,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:39,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:39,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:39,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:39,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:40,322 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:22:40,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:40,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:40,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:40,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:40,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:40,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:40,582 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:40,583 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:40,585 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:40,586 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:40,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:40,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:40,588 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:40,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:40,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:40,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:40,697 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:40,697 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:40,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:40,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:40,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:40,702 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:40,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:40,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:40,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:40,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:40,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:40,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:40,709 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:40,709 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:40,710 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:40,710 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:41,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:41,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:41,626 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:41,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:41,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:41,628 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:41,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:41,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:41,632 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:41,632 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:41,633 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:41,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:41,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:41,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:41,636 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:41,636 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:42,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:22:42,238 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:22:42,323 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:22:42,436 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:42,437 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:42,440 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:42,440 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:42,441 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:42,441 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:42,442 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:42,443 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:42,445 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:42,445 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:42,447 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:42,447 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:42,448 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:42,449 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:42,450 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:42,450 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:42,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:42,534 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:42,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:42,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:42,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:42,542 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:42,542 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:42,542 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:42,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:42,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:42,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:42,544 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:42,545 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:42,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:42,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:42,547 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:43,504 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:43,505 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:43,508 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:43,509 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:43,509 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:43,510 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:43,511 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:43,511 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:43,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:43,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:43,515 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:43,516 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:43,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:43,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:43,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:43,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:44,194 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:44,194 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:44,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:44,198 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:44,198 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:44,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:44,200 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:44,200 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:44,202 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:44,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:44,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:44,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:44,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:44,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:44,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:44,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:44,302 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:44,302 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:44,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:44,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:44,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:44,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:44,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:44,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:44,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:44,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:44,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:44,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:44,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:44,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:44,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:44,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:44,324 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:22:44,447 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:44,448 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:44,451 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:44,451 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:44,452 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:44,452 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:44,453 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:44,454 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:44,456 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:44,457 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:44,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:44,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:44,459 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:44,460 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:44,460 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:44,461 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:45,077 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:45,078 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:45,081 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:45,081 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:45,082 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:45,082 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:45,083 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:45,084 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:45,086 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:45,086 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:45,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:45,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:45,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:45,090 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:45,090 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:45,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:46,325 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:22:47,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:47,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:47,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:47,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:47,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:47,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:47,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:47,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:47,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:47,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:47,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:47,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:47,626 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:47,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:47,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:47,628 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:48,326 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:22:48,361 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:48,362 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:48,364 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:48,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:48,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:48,366 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:48,367 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:48,367 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:48,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:48,370 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:48,371 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:48,371 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:48,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:48,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:48,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:48,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:48,420 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:48,420 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:48,423 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:48,423 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:48,424 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:48,425 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:48,426 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:48,426 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:48,428 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:48,429 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:48,430 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:48,430 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:48,432 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:48,432 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:48,433 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:48,433 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:49,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:49,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:49,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:49,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:49,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:49,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:49,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:49,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:49,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:49,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:49,331 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:49,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:49,333 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:49,333 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:49,334 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:49,334 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:49,876 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:49,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:49,881 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:49,881 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:49,882 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:49,883 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:49,884 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:49,885 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:49,887 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:49,888 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:49,889 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:49,889 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:49,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:49,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:49,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:49,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:50,327 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:22:51,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:51,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:51,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:51,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:51,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:51,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:51,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:51,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:51,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:51,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:51,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:51,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:51,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:51,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:51,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:51,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:51,710 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:51,711 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:51,714 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:51,714 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:51,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:51,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:51,716 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:51,717 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:51,719 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:51,719 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:51,720 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:51,721 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:51,722 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:51,723 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:51,723 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:51,724 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:52,328 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:22:52,383 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:52,384 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:52,387 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:52,387 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:52,388 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:52,388 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:52,389 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:52,390 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:52,392 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:52,392 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:52,393 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:52,394 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:52,395 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:52,395 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:52,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:52,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:52,538 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:52,539 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:52,542 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:52,542 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:52,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:52,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:52,544 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:52,545 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:52,547 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:52,547 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:52,548 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:52,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:52,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:52,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:52,551 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:52,551 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:53,662 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:22:53,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:53,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:53,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:53,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:53,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:53,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:53,816 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:53,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:53,819 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:53,819 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:53,820 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:53,821 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:53,822 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:53,822 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:53,823 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:53,823 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:54,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:54,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:54,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:54,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:54,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:54,331 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:22:54,331 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:54,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:54,333 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:54,334 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:54,335 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:54,336 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:54,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:54,338 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:54,338 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:54,339 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:54,340 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:55,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:55,275 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:55,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:55,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:55,279 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:55,279 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:55,280 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:55,281 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:55,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:55,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:55,284 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:55,285 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:55,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:55,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:55,287 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:55,287 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:55,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:55,826 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:55,828 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:55,829 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:55,829 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:55,830 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:55,831 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:55,831 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:55,833 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:55,833 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:55,834 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:55,835 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:55,836 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:55,836 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:55,837 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:55,837 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:55,842 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:55,842 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:55,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:55,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:55,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:55,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:55,847 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:55,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:55,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:55,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:55,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:55,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:55,852 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:55,853 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:55,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:55,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:56,331 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:22:57,496 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:22:57,496 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:22:57,722 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:57,723 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:57,724 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:57,724 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:57,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:57,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:57,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:57,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:57,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:57,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:57,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:57,727 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:57,727 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:57,727 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:57,728 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:57,729 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:58,269 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:58,269 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:58,272 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:58,272 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:58,273 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:58,273 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:58,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:58,275 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:58,276 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:58,277 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:58,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:58,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:58,280 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:58,280 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:58,281 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:58,281 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:58,333 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:22:59,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:59,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:59,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:59,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:59,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:59,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:59,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:59,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:59,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:59,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:59,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:59,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:59,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:59,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:59,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:22:59,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,279 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,281 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,282 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,282 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,284 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,284 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,287 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,334 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:23:00,349 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,350 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,353 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,353 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,354 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,354 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,355 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,356 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,357 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,358 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,359 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,359 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,360 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,361 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,362 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,362 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,697 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,702 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,704 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,705 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,709 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,774 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,775 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,779 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,783 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,785 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,785 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,786 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:00,786 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:02,082 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:02,083 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:02,086 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:02,086 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:02,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:02,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:02,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:02,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:02,090 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:02,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:02,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:02,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:02,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:02,094 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:02,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:02,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:02,335 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:23:03,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:03,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:03,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:03,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:03,157 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:03,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:03,159 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:03,159 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:03,161 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:03,161 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:03,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:03,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:03,164 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:03,164 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:03,165 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:03,165 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:03,180 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:03,181 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:03,182 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:03,182 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:03,183 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:03,183 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:03,184 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:03,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:03,187 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:03,187 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:03,188 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:03,189 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:03,190 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:03,190 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:03,191 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:03,191 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:03,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:03,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:03,771 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:03,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:03,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:03,773 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:03,774 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:03,774 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:03,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:03,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:03,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:03,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:03,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:03,779 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:03,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:03,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:04,336 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:23:05,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:05,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:05,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:05,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:05,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:05,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:05,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:05,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:05,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:05,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:05,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:05,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:05,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:05,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:05,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:05,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:06,337 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:23:07,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:07,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:07,165 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:07,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:07,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:07,167 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:07,168 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:07,169 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:07,170 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:07,171 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:07,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:07,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:07,173 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:07,174 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:07,174 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:07,175 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:07,292 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:07,293 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:07,295 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:07,296 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:07,296 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:07,297 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:07,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:07,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:07,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:07,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:07,301 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:07,302 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:07,303 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:07,303 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:07,304 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:07,304 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:08,338 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:23:09,630 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:09,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:09,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:09,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:09,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:09,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:09,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:09,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:09,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:09,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:09,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:09,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:09,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:09,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:09,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:09,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:10,338 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:23:12,339 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:23:12,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:23:12,725 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:23:16,341 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:23:20,343 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:23:23,742 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:23:24,344 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:23:28,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:23:28,091 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:23:28,346 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:23:32,348 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:23:36,350 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:23:38,351 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:23:42,353 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:23:43,413 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:23:43,414 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:23:44,353 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:23:48,355 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:23:50,356 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:23:52,357 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:23:53,822 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:23:56,359 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:23:58,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:58,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:58,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:58,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:58,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:58,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:58,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:58,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:58,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:58,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:58,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:58,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:58,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:58,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:58,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:58,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:23:58,360 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:23:58,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:23:58,773 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:24:00,361 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:24:01,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:01,785 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:01,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:01,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:01,789 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:01,790 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:01,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:01,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:01,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:01,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:01,796 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:01,797 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:01,798 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:01,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:01,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:01,800 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:04,362 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:24:04,485 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:04,485 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:04,489 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:04,489 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:04,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:04,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:04,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:04,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:04,496 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:04,496 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:04,497 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:04,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:04,499 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:04,500 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:04,501 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:04,501 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:06,363 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:24:09,365 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:24:11,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:11,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:11,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:11,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:11,366 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:24:11,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:11,597 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:11,597 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:11,598 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:11,599 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:11,600 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:11,600 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:11,601 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:13,367 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:24:13,999 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:24:14,000 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:24:15,368 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:24:17,369 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:24:19,370 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:24:21,371 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:24:22,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:22,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:22,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:22,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:22,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:22,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:22,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:22,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:22,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:22,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:22,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:22,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:22,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:22,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:22,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:22,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:23,372 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:24:23,900 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:24:25,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:25,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:25,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:25,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:25,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:25,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:25,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:25,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:25,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:25,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:25,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:25,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:25,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:25,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:25,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:25,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:25,373 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:24:27,374 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:24:28,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:28,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:28,942 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:28,943 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:28,943 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:28,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:28,945 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:28,946 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:28,948 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:28,948 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:28,949 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:28,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:28,951 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:28,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:28,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:28,953 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:29,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:24:29,374 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:24:29,378 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:24:30,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:30,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:30,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:30,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:30,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:30,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:30,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:30,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:30,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:30,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:30,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:30,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:30,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:30,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:30,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:30,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:30,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:30,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:30,584 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:30,585 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:30,585 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:30,586 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:30,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:30,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:30,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:30,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:30,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:30,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:30,593 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:30,593 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:30,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:30,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:30,787 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:30,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:30,790 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:30,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:30,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:30,792 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:30,793 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:30,794 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:30,796 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:30,796 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:30,797 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:30,798 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:30,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:30,800 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:30,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:30,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:31,379 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:24:31,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:31,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:31,472 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:31,472 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:31,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:31,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:31,474 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:31,475 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:31,477 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:31,478 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:31,479 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:31,479 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:31,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:31,481 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:31,481 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:31,482 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:31,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:31,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:31,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:31,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:31,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:31,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:31,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:31,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:31,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:31,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:31,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:31,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:31,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:31,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:31,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:31,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:31,759 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:31,760 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:31,767 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:31,767 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:31,767 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:31,768 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:31,768 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:31,768 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:31,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:31,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:31,771 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:31,771 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:31,773 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:31,773 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:31,774 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:31,774 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:33,202 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:33,202 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:33,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:33,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:33,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:33,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:33,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:33,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:33,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:33,211 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:33,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:33,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:33,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:33,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:33,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:33,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:33,287 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:33,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:33,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:33,291 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:33,291 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:33,292 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:33,293 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:33,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:33,296 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:33,296 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:33,297 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:33,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:33,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:33,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:33,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:33,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:33,380 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:24:33,454 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:33,454 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:33,457 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:33,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:33,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:33,459 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:33,460 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:33,460 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:33,463 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:33,463 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:33,464 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:33,465 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:33,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:33,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:33,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:33,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:34,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:34,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:34,293 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:34,293 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:34,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:34,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:34,295 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:34,296 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:34,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:34,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:34,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:34,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:34,301 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:34,301 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:34,302 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:34,302 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:34,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:34,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:34,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:34,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:34,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:34,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:34,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:34,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:34,931 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:34,932 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:34,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:34,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:34,934 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:34,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:34,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:34,936 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:35,381 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:24:35,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:35,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:35,942 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:35,942 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:35,943 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:35,943 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:35,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:35,945 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:35,947 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:35,947 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:35,948 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:35,949 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:35,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:35,951 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:35,951 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:35,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:36,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:36,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:36,669 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:36,669 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:36,670 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:36,670 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:36,671 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:36,672 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:36,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:36,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:36,675 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:36,676 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:36,677 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:36,677 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:36,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:36,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:36,798 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:36,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:36,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:36,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:36,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:36,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:36,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:36,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:36,807 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:36,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:36,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:36,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:36,810 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:36,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:36,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:36,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:37,382 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:24:37,459 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:37,459 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:37,462 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:37,462 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:37,463 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:37,463 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:37,464 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:37,465 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:37,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:37,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:37,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:37,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:37,470 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:37,470 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:37,471 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:37,472 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:38,174 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:38,175 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:38,178 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:38,178 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:38,179 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:38,179 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:38,180 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:38,181 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:38,183 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:38,183 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:38,184 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:38,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:38,186 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:38,186 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:38,187 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:38,187 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:38,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:38,931 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:38,934 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:38,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:38,936 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:38,936 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:38,938 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:38,938 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:38,940 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:38,941 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:38,942 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:38,943 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:38,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:38,945 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:38,945 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:38,946 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:39,382 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:24:40,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:40,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:40,628 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:40,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:40,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:40,630 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:40,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:40,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:40,633 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:40,633 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:40,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:40,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:40,636 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:40,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:40,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:40,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:41,383 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:24:41,940 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:41,941 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:41,943 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:41,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:41,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:41,945 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:41,946 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:41,946 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:41,948 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:41,949 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:41,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:41,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:41,951 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:41,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:41,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:41,953 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:42,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:42,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:42,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:42,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:42,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:42,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:42,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:42,655 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:42,657 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:42,657 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:42,658 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:42,659 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:42,660 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:42,660 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:42,661 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:42,661 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:42,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:42,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:42,669 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:42,670 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:42,670 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:42,671 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:42,672 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:42,672 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:42,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:42,675 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:42,676 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:42,676 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:42,677 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:42,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:42,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:42,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,252 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,257 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,258 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,258 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,260 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,260 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,261 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,262 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,263 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,264 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,264 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,265 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,295 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,297 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,302 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,302 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,303 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,304 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,307 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,384 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:24:43,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,936 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,936 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,938 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,938 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,938 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,940 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,940 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,941 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,945 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,946 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,948 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,949 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,949 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,951 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,951 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,953 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,953 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,954 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,955 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,956 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,957 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,957 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,985 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,985 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,986 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,986 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,987 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,988 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,990 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,990 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,991 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,992 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,993 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,993 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,994 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:43,994 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:44,520 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:44,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:44,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:44,524 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:44,525 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:44,526 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:44,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:44,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:44,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:44,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:44,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:44,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:44,534 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:44,535 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:44,536 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:44,536 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:44,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:24:44,926 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:24:45,385 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:24:46,356 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:46,356 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:46,359 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:46,359 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:46,360 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:46,360 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:46,362 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:46,362 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:46,364 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:46,364 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:46,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:46,366 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:46,367 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:46,368 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:46,368 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:46,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:47,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:47,135 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:47,137 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:47,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:47,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:47,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:47,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:47,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:47,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:47,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:47,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:47,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:47,145 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:47,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:47,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:47,147 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:47,386 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:24:47,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:47,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:47,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:47,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:47,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:47,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:47,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:47,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:47,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:47,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:47,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:47,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:47,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:47,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:47,626 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:47,626 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:48,450 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:48,450 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:48,453 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:48,453 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:48,454 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:48,454 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:48,456 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:48,456 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:48,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:48,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:48,459 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:48,460 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:48,461 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:48,461 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:48,462 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:48,462 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:49,387 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:24:49,955 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:49,956 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:49,959 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:49,960 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:49,961 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:49,962 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:49,963 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:49,964 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:49,965 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:49,966 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:49,967 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:49,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:49,969 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:49,969 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:49,970 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:49,971 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:50,159 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:50,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:50,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:50,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:50,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:50,164 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:50,165 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:50,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:50,167 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:50,168 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:50,169 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:50,169 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:50,170 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:50,171 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:50,171 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:50,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:50,727 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:50,728 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:50,730 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:50,731 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:50,731 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:50,732 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:50,733 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:50,734 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:50,735 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:50,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:50,737 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:50,737 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:50,738 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:50,739 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:50,739 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:50,740 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:51,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:51,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:51,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:51,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:51,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:51,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:51,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:51,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:51,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:51,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:51,246 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:51,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:51,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:51,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:51,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:51,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:51,388 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:24:51,823 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:51,824 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:51,826 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:51,827 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:51,827 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:51,828 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:51,829 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:51,829 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:51,831 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:51,831 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:51,832 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:51,833 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:51,834 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:51,834 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:51,835 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:51,835 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:52,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:52,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:52,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:52,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:52,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:52,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:52,291 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:52,292 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:52,293 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:52,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:52,295 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:52,296 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:52,297 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:52,297 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:52,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:52,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:52,887 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:52,888 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:52,890 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:52,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:52,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:52,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:52,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:52,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:52,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:52,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:52,896 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:52,897 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:52,898 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:52,898 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:52,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:52,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:53,389 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:24:53,984 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:24:54,996 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:54,996 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:54,999 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:55,000 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:55,000 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:55,001 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:55,002 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:55,002 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:55,004 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:55,004 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:55,005 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:55,006 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:55,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:55,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:55,008 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:55,008 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:55,390 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:24:56,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:56,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:56,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:56,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:56,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:56,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:56,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:56,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:56,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:56,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:56,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:56,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:56,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:56,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:56,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:56,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:57,391 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:24:59,001 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:59,002 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:59,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:59,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:59,008 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:59,009 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:59,011 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:59,011 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:59,022 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:59,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:59,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:59,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:59,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:59,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:59,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:59,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:24:59,392 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:25:00,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:25:00,154 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:25:03,394 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:25:09,396 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:25:13,398 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:25:15,535 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:25:15,535 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:25:17,400 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:25:18,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:25:18,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:25:18,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:25:18,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:25:18,317 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:25:18,317 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:25:18,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:25:18,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:25:18,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:25:18,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:25:18,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:25:18,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:25:18,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:25:18,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:25:18,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:25:18,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:25:19,401 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:25:23,402 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:25:24,065 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:25:27,404 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:25:29,405 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:25:30,792 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:25:30,793 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:25:33,406 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:25:35,407 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:25:40,409 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:25:42,410 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:25:46,016 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:25:46,016 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:25:46,412 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:25:48,412 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:25:50,413 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:25:50,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:25:50,544 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:25:50,547 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:25:50,547 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:25:50,548 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:25:50,548 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:25:50,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:25:50,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:25:50,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:25:50,554 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:25:50,555 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:25:50,556 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:25:50,557 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:25:50,558 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:25:50,558 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:25:50,559 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:25:52,414 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:25:54,144 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:25:54,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:25:54,491 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:25:54,494 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:25:54,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:25:54,496 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:25:54,496 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:25:54,497 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:25:54,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:25:54,501 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:25:54,501 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:25:54,502 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:25:54,503 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:25:54,504 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:25:54,505 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:25:54,505 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:25:54,506 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:25:56,416 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:25:58,417 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:25:59,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:25:59,800 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:25:59,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:25:59,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:00,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:00,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:00,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:00,284 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:00,285 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:00,285 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:00,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:00,287 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:00,418 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:26:00,463 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:00,555 DEBUG SenderThread:2194005 [sender.py:send():232] send: history +2022-10-25 18:26:00,781 DEBUG SenderThread:2194005 [sender.py:send():232] send: summary +2022-10-25 18:26:00,855 INFO SenderThread:2194005 [sender.py:_save_file():946] saving file wandb-summary.json with policy end +2022-10-25 18:26:01,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:26:01,316 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:26:01,419 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/wandb-summary.json +2022-10-25 18:26:02,419 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:26:04,420 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:26:06,421 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:26:06,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:06,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:06,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:06,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:06,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:06,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:06,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:06,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:06,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:06,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:06,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:06,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:06,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:06,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:06,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:06,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:08,422 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:26:10,423 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:26:12,424 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:26:13,762 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:13,763 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:13,766 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:13,766 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:13,767 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:13,768 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:13,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:13,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:13,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:13,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:13,773 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:13,774 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:13,775 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:13,775 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:13,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:13,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:14,425 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:26:16,425 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:26:16,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:26:16,568 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:26:18,426 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:26:19,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:19,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:19,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:19,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:19,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:19,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:19,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:19,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:19,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:19,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:19,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:19,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:19,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:19,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:19,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:19,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:20,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:20,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:20,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:20,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:20,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:20,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:20,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:20,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:20,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:20,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:20,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:20,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:20,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:20,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:20,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:20,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:20,427 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:26:21,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:21,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:21,598 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:21,598 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:21,599 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:21,599 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:21,600 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:21,601 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:21,603 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:21,603 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:21,604 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:21,605 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:21,606 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:21,606 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:21,607 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:21,607 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:22,428 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:26:22,488 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:22,489 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:22,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:22,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:22,493 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:22,493 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:22,494 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:22,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:22,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:22,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:22,499 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:22,500 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:22,501 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:22,501 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:22,502 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:22,502 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:22,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:22,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:22,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:22,626 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:22,626 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:22,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:22,628 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:22,628 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:22,630 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:22,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:22,632 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:22,633 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:22,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:22,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:22,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:22,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:22,687 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:22,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:22,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:22,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:22,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:22,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:22,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:22,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:22,697 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:22,697 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:22,697 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:22,698 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:22,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:22,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:22,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:22,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:23,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:23,366 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:23,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:23,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:23,370 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:23,370 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:23,371 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:23,372 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:23,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:23,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:23,375 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:23,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:23,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:23,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:23,378 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:23,379 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:24,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:24,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:24,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:24,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:24,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:24,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:24,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:24,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:24,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:24,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:24,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:24,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:24,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:24,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:24,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:24,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:24,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:24,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:24,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:24,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:24,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:24,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:24,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:24,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:24,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:24,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:24,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:24,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:24,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:24,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:24,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:24,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:24,238 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:26:24,429 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:26:25,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:25,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:25,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:25,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:25,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:25,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:25,145 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:25,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:25,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:25,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:25,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:25,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:25,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:25,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:25,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:25,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:25,826 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:25,826 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:25,829 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:25,829 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:25,830 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:25,831 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:25,832 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:25,832 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:25,834 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:25,835 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:25,836 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:25,836 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:25,837 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:25,838 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:25,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:25,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:26,430 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:26:26,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:26,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:26,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:26,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:26,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:26,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:26,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:26,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:26,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:26,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:26,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:26,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:26,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:26,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:26,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:26,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:27,470 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:27,471 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:27,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:27,474 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:27,474 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:27,475 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:27,476 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:27,476 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:27,478 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:27,479 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:27,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:27,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:27,481 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:27,482 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:27,483 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:27,483 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:27,607 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:27,607 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:27,610 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:27,611 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:27,611 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:27,612 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:27,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:27,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:27,616 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:27,616 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:27,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:27,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:27,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:27,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:27,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:27,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:28,001 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:28,001 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:28,004 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:28,005 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:28,006 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:28,006 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:28,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:28,008 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:28,010 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:28,010 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:28,011 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:28,012 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:28,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:28,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:28,014 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:28,015 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:28,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:28,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:28,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:28,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:28,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:28,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:28,155 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:28,155 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:28,157 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:28,157 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:28,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:28,159 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:28,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:28,161 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:28,161 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:28,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:28,430 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:26:28,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:28,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:28,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:28,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:28,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:28,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:28,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:28,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:28,932 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:28,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:28,934 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:28,934 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:28,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:28,936 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:28,936 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:28,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:29,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:29,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:29,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:29,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:29,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:29,090 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:29,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:29,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:29,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:29,094 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:29,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:29,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:29,096 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:29,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:29,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:29,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:30,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:30,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:30,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:30,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:30,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:30,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:30,331 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:30,331 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:30,333 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:30,333 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:30,334 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:30,335 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:30,336 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:30,336 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:30,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:30,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:30,431 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:26:31,842 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:26:31,842 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:26:32,368 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:32,368 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:32,371 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:32,372 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:32,372 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:32,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:32,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:32,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:32,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:32,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:32,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:32,378 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:32,379 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:32,379 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:32,380 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:32,380 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:32,432 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:26:32,907 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:32,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:32,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:32,912 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:32,913 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:32,914 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:32,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:32,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:32,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:32,919 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:32,920 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:32,921 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:32,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:32,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:32,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:32,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,598 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,599 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,601 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,601 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,602 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,602 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,603 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,604 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,606 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,606 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,607 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,608 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,609 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,609 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,610 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,610 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,645 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,647 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,671 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,671 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,675 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,675 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,676 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,677 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,680 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,680 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,681 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,682 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,682 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,683 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,722 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,722 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,727 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,728 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,730 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,730 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,731 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,732 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,733 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,733 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,734 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:33,734 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:34,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:34,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:34,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:34,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:34,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:34,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:34,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:34,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:34,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:34,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:34,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:34,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:34,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:34,331 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:34,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:34,333 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:34,433 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:26:34,446 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:34,447 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:34,449 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:34,450 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:34,450 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:34,451 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:34,452 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:34,453 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:34,454 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:34,455 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:34,456 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:34,456 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:34,457 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:34,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:34,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:34,459 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:35,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:35,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:35,556 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:35,557 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:35,558 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:35,559 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:35,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:35,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:35,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:35,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:35,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:35,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:35,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:35,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:35,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:35,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:36,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:36,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:36,211 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:36,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:36,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:36,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:36,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:36,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:36,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:36,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:36,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:36,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:36,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:36,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:36,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:36,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:36,434 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:26:36,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:36,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:36,807 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:36,807 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:36,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:36,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:36,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:36,810 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:36,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:36,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:36,813 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:36,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:36,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:36,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:36,816 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:36,816 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:37,453 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:37,453 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:37,456 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:37,456 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:37,457 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:37,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:37,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:37,459 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:37,461 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:37,461 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:37,462 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:37,463 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:37,464 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:37,464 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:37,465 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:37,465 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:37,992 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:37,992 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:37,995 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:37,995 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:37,996 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:37,996 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:37,997 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:37,998 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:38,000 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:38,000 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:38,001 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:38,002 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:38,003 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:38,003 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:38,004 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:38,004 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:38,435 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:26:39,176 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:39,177 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:39,184 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:39,184 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:39,184 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:39,184 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:39,184 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:39,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:39,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:39,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:39,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:39,186 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:39,187 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:39,187 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:39,188 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:39,188 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:39,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:39,632 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:39,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:39,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:39,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:39,636 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:39,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:39,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:39,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:39,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:39,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:39,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:39,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:39,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:39,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:39,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:40,436 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:26:40,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:40,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:40,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:40,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:40,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:40,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:40,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:40,709 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:40,710 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:40,711 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:40,712 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:40,712 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:40,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:40,714 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:40,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:40,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:40,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:40,792 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:40,794 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:40,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:40,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:40,796 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:40,797 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:40,798 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:40,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:40,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:40,800 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:40,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:40,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:40,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:40,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:40,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:41,307 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:41,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:41,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:41,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:41,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:41,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:41,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:41,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:41,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:41,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:41,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:41,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:41,317 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:41,318 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:41,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:41,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:41,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:41,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:41,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:41,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:41,645 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:41,645 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:41,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:41,647 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:41,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:41,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:41,650 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:41,650 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:41,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:41,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:41,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:41,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:42,436 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:26:43,266 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:43,266 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:43,269 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:43,269 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:43,270 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:43,270 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:43,271 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:43,272 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:43,273 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:43,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:43,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:43,275 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:43,276 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:43,277 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:43,277 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:43,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:43,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:43,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:43,698 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:43,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:43,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:43,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:43,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:43,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:43,702 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:43,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:43,704 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:43,705 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:43,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:43,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:43,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:43,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:44,438 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:26:45,178 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:45,178 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:45,181 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:45,181 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:45,182 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:45,182 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:45,183 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:45,183 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:45,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:45,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:45,186 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:45,187 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:45,188 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:45,188 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:45,189 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:45,189 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:46,439 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:26:47,067 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:26:47,068 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:26:48,439 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:26:49,793 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:49,793 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:49,798 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:49,798 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:49,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:49,800 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:49,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:49,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:49,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:49,810 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:49,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:49,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:49,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:49,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:49,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:49,816 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:52,441 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:26:54,316 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:26:56,443 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:26:57,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:57,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:57,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:57,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:57,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:57,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:57,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:57,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:57,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:57,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:57,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:57,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:57,717 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:57,717 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:57,718 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:26:57,719 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:00,444 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:27:02,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:27:02,411 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:27:04,446 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:27:06,447 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:27:10,449 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:27:14,450 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:27:14,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:14,675 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:14,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:14,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:14,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:14,680 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:14,681 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:14,682 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:14,686 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:14,686 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:14,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:14,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:14,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:14,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:14,691 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:14,691 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:16,451 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:27:17,668 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:27:17,669 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:27:20,453 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:27:22,454 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:27:24,395 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:27:26,456 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:27:28,457 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:27:30,458 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:27:32,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:27:32,923 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:27:34,460 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:27:36,461 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:27:38,462 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:27:40,463 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:27:42,464 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:27:43,512 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:43,513 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:43,516 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:43,516 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:43,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:43,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:43,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:43,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:43,522 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:43,522 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:43,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:43,524 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:43,525 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:43,526 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:43,526 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:43,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:45,750 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:45,751 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:45,758 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:45,758 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:46,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:46,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:46,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:46,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:46,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:46,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:46,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:46,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:46,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:46,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:46,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:46,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:46,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:46,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:46,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:46,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:46,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:46,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:46,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:46,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:46,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:46,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:46,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:46,257 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:46,465 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:27:48,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:27:48,255 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:27:48,466 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:27:50,467 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:27:52,468 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:27:54,469 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:27:54,475 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:27:56,470 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:27:56,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:56,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:56,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:56,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:56,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:56,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:56,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:56,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:56,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:56,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:56,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:56,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:56,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:56,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:56,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:56,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:27:58,471 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:28:00,472 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:28:02,473 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:28:03,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:28:03,572 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:28:04,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:04,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:04,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:04,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:04,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:04,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:04,257 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:04,257 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:04,260 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:04,260 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:04,261 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:04,262 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:04,263 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:04,264 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:04,264 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:04,265 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:04,473 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:28:07,475 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:28:07,732 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:07,733 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:07,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:07,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:07,737 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:07,737 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:07,738 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:07,739 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:07,741 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:07,741 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:07,742 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:07,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:07,744 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:07,745 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:07,745 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:07,746 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:08,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:08,645 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:08,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:08,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:08,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:08,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:08,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:08,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:08,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:08,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:08,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:08,655 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:08,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:08,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:08,657 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:08,658 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:09,192 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:09,193 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:09,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:09,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:09,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:09,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:09,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:09,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:09,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:09,202 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:09,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:09,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:09,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:09,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:09,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:09,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:09,476 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:28:10,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:10,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:10,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:10,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:10,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:10,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:10,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:10,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:10,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:10,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:10,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:10,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:10,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:10,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:10,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:10,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:11,477 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:28:11,836 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:11,836 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:11,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:11,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:11,840 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:11,841 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:11,841 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:11,842 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:11,844 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:11,844 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:11,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:11,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:11,847 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:11,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:11,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:11,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:12,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:12,370 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:12,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:12,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:12,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:12,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:12,375 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:12,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:12,378 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:12,378 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:12,379 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:12,380 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:12,381 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:12,382 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:12,382 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:12,383 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:12,407 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:12,407 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:12,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:12,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:12,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:12,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:12,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:12,413 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:12,415 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:12,415 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:12,416 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:12,417 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:12,418 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:12,418 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:12,419 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:12,420 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:12,513 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:12,513 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:12,516 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:12,516 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:12,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:12,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:12,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:12,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:12,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:12,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:12,522 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:12,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:12,524 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:12,525 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:12,525 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:12,526 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:13,478 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:28:13,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:13,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:13,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:13,983 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:13,983 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:13,983 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:13,984 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:13,984 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:13,984 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:13,985 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:13,985 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:13,985 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:13,986 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:13,987 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:13,988 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:13,988 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:14,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:14,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:14,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:14,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:14,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:14,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:14,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:14,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:14,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:14,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:14,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:14,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:14,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:14,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:14,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:14,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:14,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:14,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:14,820 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:14,820 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:14,821 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:14,821 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:14,822 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:14,823 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:14,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:14,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:14,826 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:14,827 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:14,828 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:14,828 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:14,829 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:14,829 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:15,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:15,022 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:15,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:15,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:15,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:15,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:15,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:15,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:15,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:15,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:15,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:15,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:15,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:15,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:15,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:15,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:15,418 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:15,418 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:15,421 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:15,422 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:15,423 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:15,424 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:15,425 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:15,425 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:15,427 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:15,428 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:15,429 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:15,429 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:15,430 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:15,431 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:15,432 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:15,432 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:15,478 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:28:15,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:15,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:15,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:15,657 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:15,657 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:15,658 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:15,659 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:15,659 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:15,661 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:15,662 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:15,663 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:15,663 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:15,665 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:15,665 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:15,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:15,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:16,318 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:16,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:16,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:16,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:16,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:16,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:16,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:16,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:16,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:16,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:16,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:16,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:16,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:16,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:16,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:16,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:16,973 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:16,974 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:16,976 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:16,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:16,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:16,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:16,979 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:16,979 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:16,981 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:16,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:16,983 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:16,983 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:16,984 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:16,985 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:16,985 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:16,986 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:17,125 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:17,126 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:17,129 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:17,129 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:17,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:17,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:17,131 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:17,132 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:17,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:17,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:17,135 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:17,136 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:17,137 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:17,137 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:17,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:17,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:17,479 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:28:18,352 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:18,353 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:18,355 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:18,356 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:18,356 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:18,357 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:18,358 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:18,358 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:18,360 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:18,360 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:18,361 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:18,362 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:18,363 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:18,364 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:18,364 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:18,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:18,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:28:18,815 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:28:19,480 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:28:19,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:19,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:19,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:19,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:19,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:19,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:19,626 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:19,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:19,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:19,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:19,630 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:19,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:19,632 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:19,632 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:19,633 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:19,633 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:19,760 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:19,760 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:19,763 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:19,763 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:19,764 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:19,764 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:19,765 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:19,766 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:19,767 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:19,768 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:19,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:19,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:19,770 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:19,771 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:19,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:19,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:20,284 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:20,285 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:20,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:20,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:20,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:20,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:20,291 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:20,292 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:20,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:20,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:20,295 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:20,296 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:20,297 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:20,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:20,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:20,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:21,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:21,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:21,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:21,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:21,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:21,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:21,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:21,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:21,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:21,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:21,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:21,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:21,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:21,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:21,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:21,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:21,078 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:21,078 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:21,081 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:21,081 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:21,082 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:21,082 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:21,083 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:21,084 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:21,086 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:21,086 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:21,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:21,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:21,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:21,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:21,090 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:21,090 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:21,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:21,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:21,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:21,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:21,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:21,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:21,100 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:21,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:21,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:21,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:21,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:21,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:21,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:21,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:21,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:21,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:21,481 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:28:22,261 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:22,262 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:22,264 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:22,265 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:22,265 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:22,266 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:22,267 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:22,268 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:22,269 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:22,270 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:22,271 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:22,271 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:22,272 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:22,273 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:22,273 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:22,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:22,914 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:22,915 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:22,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:22,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:22,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:22,919 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:22,920 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:22,920 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:22,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:22,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:22,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:22,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:22,925 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:22,925 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:22,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:22,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:23,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:23,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:23,461 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:23,462 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:23,462 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:23,463 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:23,464 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:23,464 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:23,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:23,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:23,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:23,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:23,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:23,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:23,470 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:23,471 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:23,482 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:28:24,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:24,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:24,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:24,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:24,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:24,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:24,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:24,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:24,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:24,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:24,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:24,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:24,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:24,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:24,046 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:24,046 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:24,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:24,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:24,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:24,096 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:24,096 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:24,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:24,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:24,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:24,100 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:24,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:24,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:24,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:24,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:24,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:24,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:24,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:24,554 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:28:25,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:25,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:25,246 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:25,246 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:25,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:25,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:25,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:25,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:25,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:25,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:25,252 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:25,252 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:25,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:25,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:25,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:25,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:25,482 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:28:26,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:26,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:26,318 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:26,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:26,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:26,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:26,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:26,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:26,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:26,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:26,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:26,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:26,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:26,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:26,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:26,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:27,483 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:28:27,487 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:27,487 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:27,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:27,491 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:27,491 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:27,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:27,493 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:27,493 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:27,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:27,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:27,496 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:27,497 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:27,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:27,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:27,499 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:27,499 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:27,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:27,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:27,853 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:27,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:27,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:27,855 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:27,856 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:27,856 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:27,858 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:27,858 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:27,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:27,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:27,861 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:27,861 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:27,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:27,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:27,874 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:27,875 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:27,877 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:27,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:27,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:27,879 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:27,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:27,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:27,882 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:27,882 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:27,883 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:27,884 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:27,885 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:27,885 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:27,886 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:27,886 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,405 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,405 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,408 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,409 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,413 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,414 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,415 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,416 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,417 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,418 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,419 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,419 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,420 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,484 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:28:29,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,582 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,582 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,583 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,925 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,925 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,931 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,931 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,932 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,945 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,947 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,948 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,948 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,949 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,953 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,954 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,955 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,955 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,956 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:29,956 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:30,884 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:30,885 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:30,887 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:30,887 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:30,888 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:30,888 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:30,889 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:30,890 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:30,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:30,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:30,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:30,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:30,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:30,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:30,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:30,896 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:31,485 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:28:31,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:31,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:31,717 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:31,718 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:31,718 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:31,719 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:31,720 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:31,720 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:31,721 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:31,722 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:31,723 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:31,723 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:31,724 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:31,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:31,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:31,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:33,486 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:28:34,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:28:34,042 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:28:39,489 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:28:42,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:42,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:42,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:42,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:42,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:42,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:42,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:42,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:42,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:42,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:42,870 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:42,870 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:42,872 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:42,873 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:42,873 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:42,874 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:43,491 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:28:47,492 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:28:49,265 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:28:49,266 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:28:49,493 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:28:49,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:49,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:49,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:49,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:49,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:49,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:49,535 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:49,535 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:49,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:49,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:49,542 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:49,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:49,544 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:49,545 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:49,545 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:49,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:53,495 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:28:54,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:54,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:54,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:54,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:54,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:54,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:54,535 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:54,535 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:54,539 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:54,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:54,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:54,542 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:54,544 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:54,544 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:54,545 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:54,545 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:28:54,633 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:28:57,496 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:28:59,497 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:29:03,499 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:29:04,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:29:04,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:29:04,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:29:04,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:29:04,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:29:04,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:29:04,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:29:04,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:29:04,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:29:04,065 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:29:04,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:29:04,067 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:29:04,068 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:29:04,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:29:04,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:29:04,070 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:29:04,584 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:29:04,585 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:29:05,500 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:29:09,501 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:29:11,502 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:29:13,503 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:29:17,505 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:29:19,505 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:29:19,842 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:29:19,842 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:29:21,506 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:29:23,507 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:29:24,713 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:29:27,509 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:29:29,510 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:29:31,482 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:29:31,483 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:29:31,484 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:29:31,484 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:29:31,510 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:29:31,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:29:31,963 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:29:31,964 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:29:31,965 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:29:31,966 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:29:31,966 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:29:31,967 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:29:31,967 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:29:32,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 18:29:32,229 DEBUG SenderThread:2194005 [sender.py:send():232] send: history +2022-10-25 18:29:32,441 DEBUG SenderThread:2194005 [sender.py:send():232] send: summary +2022-10-25 18:29:32,516 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:29:32,519 INFO SenderThread:2194005 [sender.py:_save_file():946] saving file wandb-summary.json with policy end +2022-10-25 18:29:33,517 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:29:33,517 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/wandb-summary.json +2022-10-25 18:29:35,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:29:35,116 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:29:35,518 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:29:37,519 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:29:39,520 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:29:41,521 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:29:43,522 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:29:45,523 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:29:47,524 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:29:49,525 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:29:50,356 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:29:50,356 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:29:51,526 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:29:53,527 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:29:54,804 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:29:55,528 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:29:57,528 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:29:59,530 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:30:02,531 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:30:04,532 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:30:06,533 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:30:06,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:30:06,591 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:30:08,534 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:30:10,535 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:30:12,536 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:30:14,537 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:30:16,538 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:30:18,539 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:30:20,540 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:30:21,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:30:21,911 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:30:22,541 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:30:24,542 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:30:24,883 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:30:26,544 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:30:28,544 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:30:30,545 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:30:32,546 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:30:34,547 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:30:36,548 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:30:37,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:30:37,197 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:30:38,549 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:30:40,550 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:30:42,551 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:30:44,552 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:30:46,553 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:30:48,554 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:30:50,555 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:30:52,537 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:30:52,537 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:30:52,556 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:30:54,557 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:30:54,962 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:30:56,558 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:30:58,559 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:31:00,560 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:31:02,561 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:31:04,562 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:31:06,563 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:31:07,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:31:07,806 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:31:09,564 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:31:11,565 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:31:13,566 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:31:15,566 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:31:17,567 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:31:19,568 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:31:21,569 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:31:23,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:31:23,050 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:31:23,569 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:31:25,041 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:31:25,570 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:31:27,571 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:31:29,572 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:31:31,573 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:31:33,573 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:31:35,574 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:31:37,575 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:31:38,353 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:31:38,353 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:31:39,578 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:31:41,579 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:31:43,580 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:31:45,581 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:31:47,582 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:31:49,582 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:31:51,583 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:31:53,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:31:53,579 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:31:53,584 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:31:55,120 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:31:55,585 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:31:57,585 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:31:59,586 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:32:01,587 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:32:03,588 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:32:05,588 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:32:07,589 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:32:08,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:32:08,900 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:32:09,590 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:32:11,591 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:32:14,592 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:32:16,593 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:32:18,594 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:32:20,594 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:32:22,595 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:32:24,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:32:24,123 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:32:24,596 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:32:25,199 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:32:26,596 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:32:28,597 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:32:30,598 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:32:32,599 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:32:34,600 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:32:36,600 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:32:38,601 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:32:39,378 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:32:39,378 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:32:40,604 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:32:42,604 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:32:44,605 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:32:46,606 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:32:48,607 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:32:50,607 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:32:52,608 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:32:54,609 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:32:54,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:32:54,716 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:32:55,278 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:32:56,609 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:32:58,610 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:33:00,611 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:33:02,612 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:33:04,612 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:33:06,613 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:33:08,614 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:33:10,014 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:33:10,014 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:33:10,614 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:33:12,615 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:33:14,616 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:33:16,617 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:33:18,618 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:33:20,619 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:33:22,619 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:33:24,620 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:33:25,282 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:33:25,283 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:33:25,506 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:33:26,621 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:33:28,622 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:33:30,623 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:33:32,624 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:33:34,625 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:33:36,626 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:33:38,627 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:33:40,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:33:40,572 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:33:40,628 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:33:42,628 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:33:45,629 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:33:47,630 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:33:49,631 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:33:51,632 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:33:53,633 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:33:55,436 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:33:55,634 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:33:55,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:33:55,902 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:33:57,635 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:33:59,636 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:34:01,637 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:34:03,638 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:34:05,639 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:34:07,640 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:34:09,641 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:34:11,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:34:11,138 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:34:11,642 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:34:13,643 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:34:15,644 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:34:17,645 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:34:19,645 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:34:21,646 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:34:23,647 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:34:25,515 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:34:25,648 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:34:26,426 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:34:26,427 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:34:27,650 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:34:29,650 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:34:31,651 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:34:33,652 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:34:35,652 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:34:37,653 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:34:39,654 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:34:41,655 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:34:41,731 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:34:41,731 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:34:43,656 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:34:45,657 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:34:47,658 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:34:49,659 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:34:51,660 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:34:53,662 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:34:55,593 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:34:55,663 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:34:56,963 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:34:56,963 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:34:57,664 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:34:59,664 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:35:01,665 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:35:03,666 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:35:05,667 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:35:07,668 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:35:09,669 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:35:11,670 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:35:12,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:35:12,228 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:35:13,671 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:35:15,672 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:35:17,673 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:35:19,674 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:35:21,675 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:35:23,676 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:35:25,677 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:35:25,681 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:35:27,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:35:27,591 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:35:28,678 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:35:30,679 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:35:32,680 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:35:34,682 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:35:36,683 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:35:38,684 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:35:40,685 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:35:42,686 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:35:42,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:35:42,878 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:35:44,687 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:35:46,688 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:35:48,689 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:35:50,690 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:35:52,691 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:35:54,692 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:35:55,750 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:35:56,693 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:35:58,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:35:58,152 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:35:58,694 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:36:00,695 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:36:02,696 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:36:04,697 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:36:06,698 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:36:08,699 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:36:10,700 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:36:12,701 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:36:13,378 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:36:13,378 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:36:14,702 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:36:16,703 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:36:18,704 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:36:20,705 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:36:22,706 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:36:24,707 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:36:25,829 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:36:26,708 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:36:28,601 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:36:28,602 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:36:28,709 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:36:30,710 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:36:32,711 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:36:34,712 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:36:36,713 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:36:38,714 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:36:40,715 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:36:42,716 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:36:43,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:36:43,894 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:36:44,717 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:36:46,718 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:36:48,719 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:36:50,720 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:36:52,721 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:36:54,722 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:36:55,909 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:36:56,723 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:36:58,724 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:36:59,252 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:36:59,252 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:37:00,725 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:37:02,726 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:37:04,727 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:37:06,728 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:37:08,729 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:37:10,729 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:37:12,731 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:37:14,477 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:37:14,477 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:37:15,732 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:37:17,733 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:37:19,734 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:37:21,735 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:37:23,736 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:37:25,737 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:37:25,989 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:37:27,738 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:37:29,709 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:37:29,709 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:37:29,739 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:37:31,740 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:37:33,741 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:37:35,742 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:37:37,743 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:37:39,744 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:37:41,745 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:37:43,746 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:37:44,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:37:44,969 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:37:45,747 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:37:47,748 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:37:49,749 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:37:51,750 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:37:53,751 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:37:55,752 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:37:56,069 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:37:57,753 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:37:59,754 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:38:00,304 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:38:00,305 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:38:01,755 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:38:03,756 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:38:05,757 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:38:07,758 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:38:09,759 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:38:11,760 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:38:13,761 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:38:15,537 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:38:15,537 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:38:15,762 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:38:17,763 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:38:19,764 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:38:21,765 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:38:23,766 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:38:25,767 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:38:26,149 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:38:27,768 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:38:29,770 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:38:30,755 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:38:30,756 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:38:31,770 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:38:33,771 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:38:35,772 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:38:37,773 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:38:39,774 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:38:42,775 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:38:44,776 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:38:46,124 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:38:46,125 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:38:46,777 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:38:48,778 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:38:50,779 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:38:52,780 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:38:54,781 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:38:56,230 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:38:56,782 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:38:58,783 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:39:00,784 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:39:01,435 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:39:01,435 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:39:02,785 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:39:04,786 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:39:06,787 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:39:08,788 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:39:10,789 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:39:12,790 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:39:14,791 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:39:16,664 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:39:16,665 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:39:16,792 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:39:18,793 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:39:20,794 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:39:22,795 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:39:24,796 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:39:26,310 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:39:26,797 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:39:28,798 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:39:30,799 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:39:31,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:39:31,952 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:39:33,800 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:39:35,801 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:39:37,802 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:39:39,803 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:39:41,804 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:39:43,805 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:39:45,806 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:39:47,344 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:39:47,344 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:39:47,807 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:39:49,808 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:39:51,809 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:39:53,810 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:39:55,811 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:39:56,390 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:39:57,812 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:39:59,813 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:40:01,814 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:40:02,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:40:02,643 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:40:03,815 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:40:05,816 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:40:07,817 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:40:09,818 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:40:11,819 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:40:13,820 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:40:15,821 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:40:17,822 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:40:17,883 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:40:17,883 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:40:19,823 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:40:21,824 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:40:23,825 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:40:25,826 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:40:26,469 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:40:27,827 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:40:29,828 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:40:31,829 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:40:33,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:40:33,104 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:40:33,830 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:40:35,831 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:40:37,832 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:40:39,833 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:40:41,834 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:40:43,835 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:40:46,836 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:40:48,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:40:48,329 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:40:48,837 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:40:50,838 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:40:52,839 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:40:54,840 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:40:56,548 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:40:56,841 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:40:58,842 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:41:00,844 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:41:02,845 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:41:03,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:41:03,575 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:41:04,846 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:41:06,847 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:41:08,847 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:41:10,848 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:41:12,849 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:41:14,850 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:41:16,851 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:41:18,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:41:18,800 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:41:18,852 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:41:20,853 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:41:22,854 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:41:24,855 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:41:26,627 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:41:26,856 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:41:28,857 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:41:30,858 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:41:32,858 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:41:34,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:41:34,014 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:41:34,859 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:41:36,860 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:41:38,861 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:41:40,862 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:41:42,863 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:41:44,864 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:41:46,865 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:41:48,865 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:41:49,383 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:41:49,384 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:41:50,866 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:41:52,867 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:41:54,868 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:41:56,705 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:41:56,869 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:41:58,870 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:42:00,870 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:42:02,871 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:42:04,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:42:04,799 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:42:04,872 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:42:06,873 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:42:08,874 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:42:10,874 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:42:12,875 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:42:14,876 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:42:16,877 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:42:18,878 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:42:20,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:42:20,092 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:42:20,879 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:42:22,880 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:42:24,880 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:42:26,784 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:42:26,881 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:42:28,882 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:42:30,883 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:42:32,884 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:42:34,885 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:42:35,421 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:42:35,421 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:42:37,886 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:42:39,887 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:42:41,888 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:42:43,889 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:42:45,889 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:42:47,890 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:42:49,891 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:42:50,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:42:50,737 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:42:51,892 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:42:53,893 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:42:55,893 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:42:56,862 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:42:57,894 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:42:59,895 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:43:01,896 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:43:03,896 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:43:05,897 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:43:06,075 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:43:06,075 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:43:07,898 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:43:09,899 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:43:11,900 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:43:13,900 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:43:15,901 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:43:17,902 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:43:19,903 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:43:21,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:43:21,313 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:43:21,904 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:43:23,904 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:43:25,905 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:43:26,940 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:43:27,906 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:43:29,907 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:43:31,908 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:43:33,909 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:43:35,910 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:43:36,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:43:36,550 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:43:37,910 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:43:39,911 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:43:41,912 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:43:43,913 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:43:45,914 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:43:47,914 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:43:49,915 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:43:51,857 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:43:51,857 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:43:51,916 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:43:53,917 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:43:55,917 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:43:57,020 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:43:57,918 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:43:59,919 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:44:01,920 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:44:03,921 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:44:05,922 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:44:07,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:44:07,128 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:44:08,923 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:44:10,923 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:44:12,924 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:44:14,925 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:44:16,926 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:44:18,927 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:44:20,928 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:44:22,404 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:44:22,404 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:44:22,929 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:44:24,930 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:44:26,930 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:44:27,100 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:44:28,931 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:44:30,932 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:44:32,933 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:44:34,934 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:44:36,934 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:44:37,632 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:44:37,632 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:44:38,935 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:44:40,936 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:44:42,937 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:44:44,938 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:44:46,938 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:44:48,939 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:44:50,940 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:44:52,941 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:44:52,984 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:44:52,984 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:44:54,942 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:44:57,180 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:44:57,943 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:44:59,943 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:45:01,944 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:45:03,945 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:45:05,946 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:45:07,947 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:45:08,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:45:08,210 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:45:09,948 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:45:11,948 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:45:13,949 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:45:15,950 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:45:17,951 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:45:19,951 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:45:21,952 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:45:23,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:45:23,480 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:45:23,953 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:45:25,954 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:45:27,259 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:45:27,955 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:45:29,956 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:45:31,956 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:45:33,957 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:45:35,958 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:45:37,959 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:45:38,697 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:45:38,698 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:45:39,960 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:45:41,961 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:45:43,961 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:45:45,962 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:45:47,963 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:45:49,964 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:45:51,965 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:45:53,966 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:45:54,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:45:54,134 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:45:55,966 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:45:57,339 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:45:57,967 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:45:59,968 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:46:01,969 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:46:03,969 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:46:05,970 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:46:07,971 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:46:09,357 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:46:09,357 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:46:09,972 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:46:11,972 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:46:13,973 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:46:15,974 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:46:17,975 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:46:19,975 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:46:21,976 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:46:24,676 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:46:24,676 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:46:24,977 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:46:26,978 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:46:27,418 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:46:28,979 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:46:30,980 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:46:32,980 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:46:34,981 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:46:36,982 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:46:38,983 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:46:39,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:46:39,924 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:46:40,984 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:46:42,985 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:46:44,985 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:46:46,986 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:46:48,987 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:46:50,988 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:46:52,989 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:46:54,989 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:46:55,155 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:46:55,155 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:46:57,497 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:46:57,991 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:46:59,991 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:47:01,992 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:47:03,993 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:47:05,994 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:47:07,995 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:47:09,996 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:47:10,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:47:10,374 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:47:11,996 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:47:13,997 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:47:15,998 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:47:17,999 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:47:20,000 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:47:22,001 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:47:24,001 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:47:25,663 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:47:25,663 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:47:26,002 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:47:27,576 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:47:28,003 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:47:30,004 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:47:32,005 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:47:34,006 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:47:36,007 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:47:38,008 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:47:40,009 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:47:40,882 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:47:40,882 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:47:42,009 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:47:44,010 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:47:46,011 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:47:48,012 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:47:50,012 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:47:52,013 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:47:54,014 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:47:56,015 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:47:56,271 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:47:56,271 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:47:57,654 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:47:58,016 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:48:00,016 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:48:02,017 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:48:04,018 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:48:06,019 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:48:09,020 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:48:11,021 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:48:11,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:48:11,570 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:48:13,022 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:48:15,023 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:48:17,024 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:48:19,025 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:48:21,026 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:48:23,027 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:48:25,028 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:48:26,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:48:26,792 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:48:27,029 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:48:27,732 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:48:29,030 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:48:31,030 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:48:33,031 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:48:35,032 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:48:37,033 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:48:39,033 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:48:41,034 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:48:42,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:48:42,090 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:48:43,035 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:48:45,036 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:48:47,037 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:48:49,038 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:48:51,039 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:48:53,040 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:48:55,041 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:48:57,042 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:48:57,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:48:57,374 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:48:57,812 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:48:59,043 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:49:01,043 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:49:03,044 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:49:05,045 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:49:07,046 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:49:09,047 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:49:11,048 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:49:12,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:49:12,593 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:49:13,049 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:49:15,050 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:49:17,051 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:49:19,052 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:49:21,052 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:49:23,053 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:49:26,054 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:49:27,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:49:27,846 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:49:28,055 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:49:28,692 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:49:30,056 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:49:32,057 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:49:34,058 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:49:36,059 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:49:38,060 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:49:40,061 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:49:42,061 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:49:43,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:49:43,703 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:49:44,062 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:49:46,063 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:49:48,064 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:49:50,065 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:49:52,066 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:49:54,066 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:49:56,067 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:49:57,972 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:49:58,068 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:49:58,921 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:49:58,921 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:50:00,069 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:50:02,070 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:50:04,071 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:50:06,072 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:50:08,072 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:50:10,073 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:50:12,074 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:50:14,075 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:50:14,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:50:14,203 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:50:16,076 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:50:18,076 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:50:20,077 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:50:22,078 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:50:24,079 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:50:26,080 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:50:28,051 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:50:29,082 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:50:29,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:50:29,552 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:50:31,083 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:50:33,084 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:50:35,085 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:50:37,086 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:50:39,086 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:50:41,087 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:50:43,088 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:50:44,856 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:50:44,857 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:50:45,089 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:50:47,090 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:50:49,091 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:50:51,091 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:50:53,093 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:50:55,093 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:50:57,094 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:50:58,135 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:50:59,095 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:51:00,082 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:51:00,083 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:51:01,096 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:51:03,096 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:51:05,097 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:51:07,098 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:51:09,099 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:51:11,100 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:51:13,101 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:51:15,101 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:51:15,297 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:51:15,297 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:51:17,102 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:51:19,103 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:51:21,104 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:51:23,105 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:51:25,106 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:51:27,107 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:51:28,214 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:51:29,108 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:51:30,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:51:30,580 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:51:31,109 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:51:33,110 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:51:35,111 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:51:37,112 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:51:39,113 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:51:41,113 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:51:43,114 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:51:45,115 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:51:45,866 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:51:45,866 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:51:47,116 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:51:49,116 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:51:51,117 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:51:53,118 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:51:55,119 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:51:57,120 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:51:58,292 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:51:59,121 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:52:01,094 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:52:01,094 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:52:01,121 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:52:03,122 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:52:05,123 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:52:07,124 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:52:09,125 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:52:11,125 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:52:13,126 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:52:15,127 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:52:16,362 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:52:16,362 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:52:17,128 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:52:19,129 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:52:21,129 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:52:24,131 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:52:26,132 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:52:28,133 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:52:28,371 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:52:30,134 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:52:31,588 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:52:31,589 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:52:32,134 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:52:34,135 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:52:36,136 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:52:38,137 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:52:40,137 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:52:42,138 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:52:44,139 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:52:46,140 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:52:46,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:52:46,879 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:52:48,141 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:52:50,142 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:52:52,142 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:52:54,143 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:52:56,144 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:52:58,145 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:52:58,451 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:53:00,146 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:53:02,146 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:53:02,174 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:53:02,175 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:53:04,147 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:53:06,148 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:53:08,149 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:53:10,150 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:53:12,150 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:53:14,151 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:53:16,152 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:53:17,443 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:53:17,443 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:53:18,153 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:53:20,154 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:53:22,155 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:53:24,156 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:53:26,157 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:53:28,157 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:53:28,530 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:53:30,158 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:53:32,159 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:53:32,667 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:53:32,667 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:53:34,160 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:53:36,161 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:53:38,162 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:53:40,162 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:53:42,163 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:53:45,164 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:53:47,165 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:53:47,963 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:53:47,963 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:53:49,166 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:53:51,167 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:53:53,168 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:53:55,169 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:53:57,169 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:53:58,610 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:53:59,170 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:54:01,171 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:54:03,172 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:54:03,262 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:54:03,263 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:54:05,173 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:54:07,173 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:54:09,174 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:54:11,175 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:54:13,176 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:54:15,177 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:54:17,178 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:54:18,485 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:54:18,486 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:54:19,178 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:54:21,179 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:54:23,180 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:54:25,181 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:54:27,181 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:54:28,691 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:54:29,182 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:54:31,183 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:54:33,184 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:54:33,879 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:54:33,879 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:54:35,185 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:54:37,185 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:54:39,186 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:54:41,187 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:54:43,188 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:54:45,189 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:54:47,190 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:54:49,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:54:49,108 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:54:49,191 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:54:51,191 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:54:53,192 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:54:55,193 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:54:57,194 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:54:58,770 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:54:59,195 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:55:01,196 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:55:03,197 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:55:04,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:55:04,458 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:55:06,198 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:55:08,199 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:55:10,200 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:55:12,201 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:55:14,202 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:55:16,203 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:55:18,204 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:55:19,685 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:55:19,685 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:55:20,204 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:55:22,205 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:55:24,206 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:55:26,207 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:55:28,208 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:55:28,850 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:55:30,208 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:55:32,209 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:55:34,210 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:55:34,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:55:34,908 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:55:36,211 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:55:38,212 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:55:40,213 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:55:42,213 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:55:44,214 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:55:46,215 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:55:48,216 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:55:50,135 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:55:50,136 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:55:50,216 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:55:52,217 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:55:54,218 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:55:56,219 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:55:58,219 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:55:58,930 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:56:00,220 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:56:02,221 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:56:04,222 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:56:05,362 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:56:05,362 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:56:06,223 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:56:08,224 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:56:10,225 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:56:12,226 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:56:14,226 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:56:16,227 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:56:18,228 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:56:20,229 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:56:20,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:56:20,587 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:56:22,230 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:56:25,231 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:56:27,232 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:56:29,010 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:56:29,233 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:56:31,234 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:56:33,234 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:56:35,235 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:56:35,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:56:35,917 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:56:37,236 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:56:39,237 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:56:41,238 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:56:43,239 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:56:45,240 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:56:47,241 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:56:49,241 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:56:51,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:56:51,142 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:56:51,242 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:56:53,243 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:56:55,244 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:56:57,245 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:56:59,089 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:56:59,246 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:57:01,247 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:57:03,248 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:57:05,248 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:57:06,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:57:06,365 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:57:07,249 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:57:09,250 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:57:11,251 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:57:13,252 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:57:15,253 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:57:17,254 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:57:19,254 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:57:21,255 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:57:21,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:57:21,643 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:57:23,256 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:57:25,258 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:57:27,258 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:57:29,168 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:57:29,259 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:57:31,260 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:57:34,261 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:57:36,262 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:57:36,865 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:57:36,865 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:57:38,263 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:57:40,263 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:57:42,264 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:57:44,265 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:57:46,266 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:57:48,267 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:57:50,268 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:57:52,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:57:52,112 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:57:52,268 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:57:54,269 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:57:56,270 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:57:58,271 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:57:59,248 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:58:00,272 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:58:02,272 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:58:04,273 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:58:06,274 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:58:07,394 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:58:07,395 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:58:08,275 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:58:10,276 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:58:12,276 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:58:14,277 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:58:16,278 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:58:18,279 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:58:20,280 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:58:22,281 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:58:22,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:58:22,643 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:58:24,281 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:58:26,282 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:58:28,283 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:58:29,327 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:58:30,284 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:58:32,284 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:58:34,285 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:58:36,286 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:58:37,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:58:37,894 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:58:38,287 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:58:40,287 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:58:42,288 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:58:44,289 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:58:46,290 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:58:48,290 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:58:50,291 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:58:52,292 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:58:53,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:58:53,117 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:58:54,293 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:58:56,294 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:58:58,294 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:58:59,407 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:59:00,295 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:59:02,296 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:59:05,297 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:59:07,298 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:59:08,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:59:08,533 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:59:09,299 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:59:11,300 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:59:13,300 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:59:15,301 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:59:17,302 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:59:19,302 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:59:21,303 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:59:23,304 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:59:23,816 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:59:23,816 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:59:25,305 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:59:27,306 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:59:29,306 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:59:29,486 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 18:59:31,307 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:59:33,308 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:59:35,309 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:59:37,310 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:59:39,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:59:39,038 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:59:39,310 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:59:41,311 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:59:43,312 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:59:45,313 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:59:47,314 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:59:49,314 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:59:51,315 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:59:53,316 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:59:54,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 18:59:54,325 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 18:59:55,317 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:59:57,318 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:59:59,319 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 18:59:59,565 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:00:02,320 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:00:04,321 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:00:06,322 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:00:08,322 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:00:09,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:00:09,616 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:00:10,323 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:00:12,324 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:00:14,325 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:00:16,326 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:00:18,327 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:00:20,327 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:00:22,328 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:00:24,329 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:00:24,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:00:24,902 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:00:26,330 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:00:28,331 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:00:29,646 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:00:30,332 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:00:32,332 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:00:34,333 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:00:36,334 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:00:38,335 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:00:40,125 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:00:40,126 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:00:40,336 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:00:42,337 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:00:44,337 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:00:46,338 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:00:48,339 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:00:50,340 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:00:52,341 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:00:54,341 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:00:55,558 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:00:55,558 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:00:56,342 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:00:58,343 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:00:59,726 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:01:00,344 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:01:02,345 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:01:04,346 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:01:06,346 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:01:08,347 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:01:10,348 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:01:10,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:01:10,799 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:01:12,349 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:01:14,350 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:01:17,351 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:01:19,352 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:01:21,352 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:01:23,353 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:01:25,354 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:01:26,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:01:26,025 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:01:27,355 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:01:29,356 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:01:29,807 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:01:31,356 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:01:33,357 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:01:35,358 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:01:37,359 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:01:39,360 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:01:41,318 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:01:41,318 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:01:41,360 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:01:43,361 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:01:45,362 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:01:47,363 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:01:49,364 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:01:51,364 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:01:53,365 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:01:55,366 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:01:56,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:01:56,636 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:01:57,367 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:01:59,368 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:01:59,889 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:02:01,368 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:02:03,369 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:02:05,370 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:02:08,371 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:02:10,372 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:02:11,897 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:02:11,897 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:02:12,373 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:02:14,374 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:02:16,375 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:02:18,375 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:02:20,376 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:02:22,377 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:02:24,378 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:02:26,379 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:02:27,170 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:02:27,170 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:02:28,380 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:02:29,972 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:02:30,380 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:02:32,381 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:02:34,382 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:02:36,383 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:02:38,384 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:02:40,385 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:02:42,385 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:02:42,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:02:42,396 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:02:44,386 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:02:46,387 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:02:48,388 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:02:50,389 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:02:52,390 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:02:54,390 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:02:56,391 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:02:57,767 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:02:57,767 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:02:58,392 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:03:00,052 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:03:00,393 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:03:02,394 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:03:04,395 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:03:06,396 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:03:08,396 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:03:10,397 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:03:12,398 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:03:13,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:03:13,045 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:03:14,399 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:03:16,400 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:03:18,400 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:03:20,401 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:03:22,402 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:03:24,403 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:03:26,404 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:03:28,270 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:03:28,270 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:03:28,405 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:03:30,133 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:03:30,406 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:03:32,406 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:03:34,407 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:03:36,408 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:03:38,409 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:03:40,410 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:03:42,411 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:03:43,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:03:43,495 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:03:44,412 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:03:46,412 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:03:48,413 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:03:51,414 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:03:53,415 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:03:55,416 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:03:57,417 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:03:58,721 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:03:58,722 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:03:59,418 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:04:00,213 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:04:01,418 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:04:03,419 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:04:05,420 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:04:07,421 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:04:09,422 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:04:11,423 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:04:13,424 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:04:14,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:04:14,057 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:04:15,424 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:04:17,425 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:04:19,426 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:04:21,427 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:04:23,428 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:04:25,429 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:04:27,430 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:04:29,360 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:04:29,360 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:04:29,431 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:04:30,293 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:04:31,432 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:04:33,433 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:04:35,434 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:04:37,435 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:04:39,436 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:04:41,437 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:04:43,438 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:04:44,586 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:04:44,587 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:04:45,439 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:04:47,440 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:04:49,441 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:04:51,442 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:04:53,443 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:04:55,444 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:04:57,444 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:04:59,445 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:04:59,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:04:59,900 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:05:00,373 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:05:01,446 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:05:03,447 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:05:05,448 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:05:07,449 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:05:09,450 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:05:11,451 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:05:13,452 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:05:15,124 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:05:15,124 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:05:15,453 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:05:18,454 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:05:20,455 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:05:22,456 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:05:24,457 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:05:26,458 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:05:28,459 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:05:30,382 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:05:30,383 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:05:30,460 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:05:30,627 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:05:32,461 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:05:34,462 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:05:36,463 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:05:38,464 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:05:40,465 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:05:42,466 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:05:44,467 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:05:45,781 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:05:45,781 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:05:46,468 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:05:48,469 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:05:50,470 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:05:52,471 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:05:54,472 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:05:56,473 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:05:58,474 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:06:00,475 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:06:00,534 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:06:01,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:06:01,013 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:06:02,476 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:06:04,477 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:06:06,478 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:06:08,479 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:06:10,480 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:06:12,481 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:06:14,482 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:06:16,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:06:16,375 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:06:16,482 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:06:18,483 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:06:20,485 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:06:22,486 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:06:24,486 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:06:26,487 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:06:28,488 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:06:30,489 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:06:30,613 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:06:31,603 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:06:31,603 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:06:32,490 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:06:34,491 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:06:36,492 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:06:38,493 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:06:40,494 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:06:42,495 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:06:44,496 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:06:46,497 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:06:46,882 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:06:46,883 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:06:48,498 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:06:50,499 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:06:53,500 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:06:55,501 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:06:57,502 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:06:59,503 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:07:00,692 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:07:01,504 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:07:02,171 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:07:02,171 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:07:03,505 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:07:05,506 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:07:07,507 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:07:09,508 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:07:11,509 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:07:13,510 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:07:15,511 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:07:17,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:07:17,492 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:07:17,512 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:07:19,513 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:07:21,514 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:07:23,515 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:07:25,516 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:07:27,517 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:07:29,518 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:07:30,772 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:07:31,519 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:07:32,790 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:07:32,790 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:07:33,520 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:07:35,521 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:07:37,522 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:07:39,523 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:07:41,523 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:07:43,524 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:07:45,525 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:07:47,526 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:07:48,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:07:48,030 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:07:49,528 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:07:51,529 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:07:53,530 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:07:55,530 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:07:57,531 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:07:59,532 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:08:00,850 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:08:01,533 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:08:03,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:08:03,326 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:08:03,534 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:08:05,535 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:08:07,536 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:08:09,537 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:08:11,538 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:08:13,539 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:08:15,540 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:08:17,541 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:08:18,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:08:18,620 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:08:19,542 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:08:21,543 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:08:23,544 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:08:25,545 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:08:27,546 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:08:29,547 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:08:30,930 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:08:31,548 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:08:33,549 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:08:33,841 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:08:33,841 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:08:35,550 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:08:38,551 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:08:40,552 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:08:42,553 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:08:44,554 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:08:46,555 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:08:48,556 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:08:49,175 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:08:49,175 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:08:50,557 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:08:52,558 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:08:54,559 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:08:56,560 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:08:58,562 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:09:00,563 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:09:01,010 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:09:02,564 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:09:04,430 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:09:04,430 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:09:04,564 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:09:06,565 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:09:08,566 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:09:10,567 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:09:12,568 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:09:14,569 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:09:16,570 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:09:18,571 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:09:19,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:09:19,726 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:09:20,572 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:09:22,573 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:09:24,574 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:09:26,575 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:09:28,576 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:09:30,578 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:09:31,089 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:09:32,578 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:09:34,579 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:09:34,947 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:09:34,947 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:09:36,580 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:09:38,581 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:09:40,582 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:09:42,583 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:09:44,584 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:09:46,585 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:09:48,586 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:09:50,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:09:50,232 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:09:50,587 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:09:52,588 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:09:54,589 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:09:56,590 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:09:58,591 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:10:00,592 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:10:01,169 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:10:02,593 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:10:04,594 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:10:05,478 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:10:05,478 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:10:06,595 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:10:08,596 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:10:10,597 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:10:12,598 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:10:14,599 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:10:16,600 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:10:18,601 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:10:20,818 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:10:20,819 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:10:21,602 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:10:23,603 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:10:25,605 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:10:27,606 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:10:29,606 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:10:31,248 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:10:31,607 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:10:33,608 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:10:35,609 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:10:36,072 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:10:36,072 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:10:37,610 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:10:39,612 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:10:41,612 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:10:43,613 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:10:45,614 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:10:47,615 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:10:49,616 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:10:51,293 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:10:51,294 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:10:51,617 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:10:53,618 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:10:55,619 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:10:57,620 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:10:59,622 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:11:01,327 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:11:01,622 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:11:03,623 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:11:05,624 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:11:06,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:11:06,514 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:11:07,625 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:11:09,626 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:11:11,627 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:11:13,628 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:11:15,629 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:11:17,630 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:11:19,631 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:11:21,632 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:11:21,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:11:21,893 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:11:23,633 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:11:25,634 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:11:27,635 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:11:29,636 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:11:31,406 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:11:31,637 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:11:33,638 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:11:35,639 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:11:37,131 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:11:37,132 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:11:37,640 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:11:39,640 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:11:42,642 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:11:44,643 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:11:46,644 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:11:48,645 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:11:50,646 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:11:52,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:11:52,468 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:11:52,647 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:11:54,648 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:11:56,649 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:11:58,650 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:12:00,651 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:12:01,486 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:12:02,652 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:12:04,653 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:12:06,654 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:12:07,691 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:12:07,691 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:12:08,655 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:12:10,656 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:12:12,657 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:12:14,658 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:12:16,659 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:12:18,660 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:12:20,661 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:12:22,662 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:12:22,919 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:12:22,919 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:12:24,663 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:12:26,664 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:12:28,665 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:12:30,666 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:12:31,565 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:12:32,667 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:12:34,668 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:12:36,669 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:12:38,135 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:12:38,136 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:12:38,670 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:12:40,671 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:12:42,672 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:12:44,673 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:12:46,674 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:12:48,675 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:12:50,676 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:12:52,676 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:12:53,392 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:12:53,392 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:12:54,677 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:12:56,678 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:12:58,679 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:13:00,680 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:13:01,645 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:13:02,681 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:13:04,682 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:13:06,683 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:13:08,738 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:13:08,738 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:13:09,684 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:13:11,685 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:13:13,686 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:13:15,687 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:13:17,688 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:13:19,689 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:13:21,690 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:13:23,691 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:13:24,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:13:24,035 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:13:25,692 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:13:27,693 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:13:29,694 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:13:31,695 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:13:31,724 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:13:33,696 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:13:35,697 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:13:37,698 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:13:39,348 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:13:39,348 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:13:39,699 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:13:41,700 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:13:43,701 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:13:45,702 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:13:47,703 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:13:49,704 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:13:51,705 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:13:53,706 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:13:54,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:13:54,644 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:13:55,707 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:13:57,708 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:13:59,709 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:14:01,710 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:14:01,803 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:14:03,711 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:14:05,712 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:14:07,713 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:14:09,714 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:14:09,943 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:14:09,943 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:14:11,715 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:14:13,715 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:14:15,716 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:14:17,717 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:14:19,717 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:14:21,718 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:14:23,720 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:14:25,167 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:14:25,167 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:14:25,720 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:14:27,721 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:14:29,722 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:14:31,723 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:14:31,882 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:14:33,724 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:14:35,725 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:14:37,726 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:14:39,727 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:14:40,436 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:14:40,436 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:14:41,728 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:14:43,729 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:14:45,730 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:14:47,731 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:14:49,732 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:14:51,733 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:14:53,734 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:14:55,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:14:55,688 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:14:55,735 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:14:58,736 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:15:00,737 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:15:01,961 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:15:02,739 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:15:04,740 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:15:06,741 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:15:08,742 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:15:10,743 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:15:10,948 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:15:10,948 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:15:12,744 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:15:14,745 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:15:16,746 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:15:18,747 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:15:20,748 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:15:22,749 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:15:24,750 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:15:26,171 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:15:26,172 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:15:26,751 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:15:28,752 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:15:30,753 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:15:32,039 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:15:32,754 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:15:34,755 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:15:36,756 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:15:38,757 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:15:40,758 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:15:41,420 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:15:41,420 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:15:42,759 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:15:44,760 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:15:46,761 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:15:48,762 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:15:50,763 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:15:52,764 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:15:54,765 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:15:56,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:15:56,641 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:15:56,766 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:15:58,767 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:16:00,768 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:16:02,119 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:16:02,769 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:16:04,770 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:16:06,771 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:16:08,772 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:16:10,773 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:16:11,986 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:16:11,986 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:16:12,774 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:16:14,775 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:16:16,776 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:16:18,777 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:16:20,778 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:16:22,779 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:16:24,780 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:16:26,780 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:16:27,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:16:27,256 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:16:28,781 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:16:30,782 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:16:32,197 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:16:32,783 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:16:34,784 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:16:36,785 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:16:38,785 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:16:40,786 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:16:42,559 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:16:42,559 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:16:42,787 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:16:44,788 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:16:46,789 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:16:48,789 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:16:50,790 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:16:52,791 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:16:54,792 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:16:57,793 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:16:57,831 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:16:57,831 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:16:59,794 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:17:01,795 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:17:02,277 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:17:03,796 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:17:05,797 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:17:07,798 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:17:09,798 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:17:11,799 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:17:13,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:17:13,056 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:17:13,800 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:17:15,801 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:17:17,802 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:17:19,803 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:17:21,804 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:17:23,804 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:17:25,805 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:17:27,806 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:17:28,494 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:17:28,494 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:17:29,807 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:17:31,808 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:17:32,357 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:17:33,809 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:17:35,809 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:17:37,810 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:17:39,811 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:17:41,812 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:17:43,737 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:17:43,737 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:17:43,813 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:17:45,813 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:17:47,814 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:17:49,815 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:17:51,816 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:17:53,817 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:17:55,817 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:17:57,818 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:17:58,953 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:17:58,953 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:17:59,819 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:18:01,820 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:18:02,438 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:18:03,821 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:18:05,822 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:18:07,822 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:18:09,823 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:18:11,824 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:18:13,825 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:18:14,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:18:14,299 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:18:15,825 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:18:17,826 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:18:19,827 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:18:21,828 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:18:23,828 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:18:26,829 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:18:28,830 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:18:29,524 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:18:29,524 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:18:30,831 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:18:32,517 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:18:32,832 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:18:34,833 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:18:36,834 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:18:38,835 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:18:40,836 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:18:42,837 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:18:44,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:18:44,806 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:18:44,837 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:18:46,839 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:18:48,840 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:18:50,841 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:18:52,841 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:18:54,842 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:18:56,843 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:18:58,844 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:19:00,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:19:00,028 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:19:00,845 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:19:02,595 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:19:02,846 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:19:04,847 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:19:06,848 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:19:08,849 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:19:10,850 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:19:12,851 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:19:14,852 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:19:15,386 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:19:15,386 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:19:16,852 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:19:18,853 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:19:20,854 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:19:22,855 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:19:24,856 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:19:26,857 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:19:28,858 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:19:30,684 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:19:30,685 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:19:30,858 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:19:32,673 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:19:32,859 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:19:34,860 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:19:36,861 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:19:38,862 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:19:40,863 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:19:42,864 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:19:44,865 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:19:45,906 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:19:45,907 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:19:46,866 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:19:49,867 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:19:51,868 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:19:53,869 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:19:55,870 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:19:57,871 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:19:59,872 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:20:01,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:20:01,205 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:20:01,873 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:20:02,753 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:20:03,874 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:20:05,875 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:20:07,876 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:20:09,876 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:20:11,877 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:20:13,878 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:20:15,879 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:20:16,435 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:20:16,435 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:20:17,880 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:20:19,881 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:20:21,882 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:20:23,883 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:20:25,884 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:20:27,884 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:20:29,885 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:20:31,718 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:20:31,718 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:20:31,886 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:20:32,831 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:20:33,887 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:20:35,888 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:20:37,888 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:20:39,889 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:20:41,890 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:20:43,891 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:20:45,892 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:20:47,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:20:47,222 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:20:47,893 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:20:49,893 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:20:51,894 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:20:53,895 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:20:55,896 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:20:57,896 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:20:59,898 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:21:01,898 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:21:02,445 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:21:02,446 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:21:02,910 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:21:03,899 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:21:05,900 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:21:07,901 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:21:09,901 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:21:11,902 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:21:13,903 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:21:15,904 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:21:17,675 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:21:17,675 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:21:17,904 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:21:19,905 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:21:21,906 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:21:23,907 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:21:25,908 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:21:27,908 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:21:29,909 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:21:32,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:21:32,892 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:21:32,911 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:21:33,121 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:21:34,912 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:21:36,912 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:21:38,913 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:21:40,914 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:21:42,915 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:21:44,916 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:21:46,917 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:21:48,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:21:48,203 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:21:48,918 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:21:50,919 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:21:52,920 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:21:54,920 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:21:56,921 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:21:58,922 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:22:00,923 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:22:02,924 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:22:03,069 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:22:03,559 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:22:03,560 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:22:04,925 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:22:06,926 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:22:08,927 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:22:10,928 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:22:12,929 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:22:14,930 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:22:16,931 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:22:18,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:22:18,805 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:22:18,932 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:22:20,933 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:22:22,934 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:22:24,935 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:22:26,936 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:22:28,937 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:22:31,939 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:22:33,149 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:22:33,940 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:22:34,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:22:34,027 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:22:35,941 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:22:37,942 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:22:39,943 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:22:41,944 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:22:43,945 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:22:45,946 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:22:47,947 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:22:49,349 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:22:49,349 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:22:49,948 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:22:51,949 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:22:53,950 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:22:55,951 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:22:57,952 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:22:59,953 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:23:01,954 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:23:03,228 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:23:03,955 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:23:04,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:23:04,572 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:23:05,956 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:23:07,957 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:23:09,958 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:23:11,959 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:23:13,960 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:23:15,961 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:23:17,962 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:23:19,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:23:19,924 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:23:19,963 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:23:22,965 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:23:24,966 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:23:26,967 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:23:28,968 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:23:30,969 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:23:32,970 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:23:33,307 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:23:34,971 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:23:35,296 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:23:35,297 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:23:36,972 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:23:38,973 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:23:40,974 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:23:42,975 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:23:44,976 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:23:46,977 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:23:48,978 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:23:50,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:23:50,529 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:23:50,979 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:23:52,980 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:23:54,981 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:23:56,982 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:23:58,983 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:24:00,984 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:24:02,985 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:24:03,386 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:24:04,986 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:24:05,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:24:05,750 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:24:06,987 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:24:08,988 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:24:10,989 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:24:12,990 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:24:14,991 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:24:16,992 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:24:18,993 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:24:20,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:24:20,969 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:24:20,994 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:24:22,995 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:24:24,996 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:24:26,997 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:24:28,998 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:24:30,999 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:24:33,000 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:24:33,465 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:24:35,001 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:24:36,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:24:36,330 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:24:37,001 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:24:39,002 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:24:41,003 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:24:43,004 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:24:46,005 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:24:48,006 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:24:50,007 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:24:51,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:24:51,587 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:24:52,007 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:24:54,008 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:24:56,009 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:24:58,010 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:25:00,011 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:25:02,012 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:25:03,544 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:25:04,013 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:25:06,013 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:25:06,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:25:06,811 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:25:08,014 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:25:10,015 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:25:12,016 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:25:14,017 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:25:16,018 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:25:18,019 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:25:20,020 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:25:22,021 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:25:22,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:25:22,219 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:25:24,022 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:25:26,023 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:25:28,024 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:25:30,025 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:25:32,026 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:25:33,623 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:25:34,026 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:25:36,027 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:25:37,442 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:25:37,442 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:25:38,028 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:25:40,029 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:25:42,030 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:25:44,031 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:25:46,032 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:25:48,033 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:25:50,034 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:25:52,035 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:25:52,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:25:52,667 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:25:54,036 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:25:56,036 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:25:58,037 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:26:00,038 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:26:02,039 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:26:03,701 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:26:04,040 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:26:06,041 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:26:07,906 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:26:07,906 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:26:08,042 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:26:10,043 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:26:12,044 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:26:15,045 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:26:17,046 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:26:19,047 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:26:21,047 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:26:23,048 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:26:23,145 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:26:23,145 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:26:25,049 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:26:27,050 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:26:29,051 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:26:31,052 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:26:33,053 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:26:33,780 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:26:35,054 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:26:37,054 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:26:38,951 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:26:38,952 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:26:39,055 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:26:41,056 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:26:43,057 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:26:45,058 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:26:47,059 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:26:49,059 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:26:51,060 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:26:53,061 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:26:54,259 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:26:54,259 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:26:55,062 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:26:57,062 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:26:59,063 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:27:01,064 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:27:03,065 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:27:03,860 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:27:05,066 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:27:07,067 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:27:09,067 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:27:09,484 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:27:09,485 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:27:11,068 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:27:13,069 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:27:15,070 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:27:17,071 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:27:19,072 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:27:21,072 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:27:23,073 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:27:24,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:27:24,846 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:27:25,074 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:27:27,075 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:27:29,075 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:27:31,076 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:27:33,077 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:27:33,940 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:27:35,078 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:27:37,079 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:27:39,080 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:27:40,183 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:27:40,184 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:27:41,080 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:27:43,081 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:27:45,082 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:27:47,083 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:27:49,084 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:27:52,085 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:27:54,086 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:27:55,471 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:27:55,471 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:27:56,087 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:27:58,087 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:28:00,088 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:28:02,089 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:28:04,018 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:28:04,090 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:28:06,091 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:28:08,091 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:28:10,092 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:28:10,797 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:28:10,797 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:28:12,093 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:28:14,094 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:28:16,095 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:28:18,095 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:28:20,096 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:28:22,097 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:28:24,098 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:28:26,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:28:26,048 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:28:26,099 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:28:28,100 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:28:30,101 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:28:32,101 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:28:34,099 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:28:34,102 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:28:36,103 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:28:38,104 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:28:40,105 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:28:41,397 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:28:41,398 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:28:42,105 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:28:44,106 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:28:46,107 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:28:48,108 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:28:50,108 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:28:52,109 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:28:54,110 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:28:56,111 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:28:56,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:28:56,702 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:28:58,112 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:29:00,112 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:29:02,113 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:29:04,114 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:29:04,179 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:29:06,115 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:29:08,116 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:29:10,116 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:29:11,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:29:11,935 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:29:12,117 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:29:14,118 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:29:16,119 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:29:18,120 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:29:20,120 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:29:22,121 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:29:24,122 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:29:27,123 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:29:27,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:29:27,158 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:29:29,124 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:29:31,125 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:29:33,126 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:29:34,259 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:29:35,127 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:29:37,128 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:29:39,129 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:29:41,130 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:29:42,547 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:29:42,547 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:29:43,131 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:29:45,132 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:29:47,133 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:29:49,134 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:29:51,134 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:29:53,135 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:29:55,136 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:29:57,137 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:29:57,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:29:57,782 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:29:59,138 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:30:01,139 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:30:03,140 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:30:04,339 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:30:05,140 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:30:07,141 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:30:09,142 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:30:11,143 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:30:13,004 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:30:13,005 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:30:13,144 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:30:15,145 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:30:17,146 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:30:19,146 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:30:21,147 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:30:23,148 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:30:25,149 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:30:27,149 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:30:28,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:30:28,365 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:30:29,150 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:30:31,151 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:30:33,152 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:30:34,418 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:30:35,153 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:30:37,153 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:30:39,154 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:30:41,155 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:30:43,156 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:30:43,730 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:30:43,730 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:30:45,157 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:30:47,157 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:30:49,158 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:30:51,159 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:30:53,160 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:30:55,161 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:30:57,161 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:30:58,997 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:30:58,998 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:31:00,163 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:31:02,163 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:31:04,164 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:31:04,498 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:31:06,165 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:31:08,166 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:31:10,166 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:31:12,167 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:31:14,168 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:31:14,284 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:31:14,285 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:31:16,169 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:31:18,169 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:31:20,170 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:31:22,171 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:31:24,172 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:31:26,172 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:31:28,173 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:31:29,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:31:29,569 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:31:30,174 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:31:32,174 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:31:34,175 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:31:34,577 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:31:36,176 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:31:38,177 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:31:40,177 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:31:42,178 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:31:44,179 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:31:44,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:31:44,796 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:31:46,180 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:31:48,180 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:31:50,181 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:31:52,182 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:31:54,183 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:31:56,184 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:31:58,185 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:32:00,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:32:00,115 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:32:00,185 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:32:02,186 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:32:04,187 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:32:04,657 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:32:06,188 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:32:08,188 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:32:10,189 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:32:12,190 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:32:14,190 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:32:15,382 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:32:15,383 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:32:16,191 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:32:18,192 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:32:20,193 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:32:22,193 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:32:24,194 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:32:26,195 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:32:28,196 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:32:30,196 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:32:30,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:32:30,613 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:32:32,197 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:32:34,739 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:32:35,198 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:32:37,199 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:32:39,200 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:32:41,201 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:32:43,202 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:32:45,203 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:32:45,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:32:45,975 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:32:47,203 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:32:49,204 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:32:51,205 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:32:53,206 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:32:55,207 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:32:57,207 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:32:59,208 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:33:01,209 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:33:01,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:33:01,288 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:33:03,210 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:33:04,817 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:33:05,210 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:33:07,211 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:33:09,212 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:33:11,212 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:33:13,213 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:33:15,214 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:33:16,544 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:33:16,544 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:33:17,214 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:33:19,215 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:33:21,216 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:33:23,217 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:33:25,217 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:33:27,218 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:33:29,219 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:33:31,220 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:33:31,800 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:33:31,801 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:33:33,221 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:33:34,898 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:33:35,222 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:33:37,222 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:33:39,223 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:33:41,224 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:33:44,225 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:33:46,226 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:33:47,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:33:47,026 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:33:48,226 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:33:50,227 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:33:52,228 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:33:54,229 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:33:56,230 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:33:58,230 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:34:00,231 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:34:02,232 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:34:02,302 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:34:02,302 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:34:04,233 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:34:04,980 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:34:06,234 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:34:08,235 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:34:10,235 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:34:12,236 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:34:14,237 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:34:16,238 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:34:17,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:34:17,561 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:34:18,239 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:34:20,239 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:34:22,240 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:34:24,241 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:34:26,242 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:34:28,243 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:34:30,243 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:34:32,244 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:34:32,785 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:34:32,786 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:34:34,245 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:34:35,059 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:34:36,246 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:34:38,247 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:34:40,247 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:34:43,249 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:34:45,249 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:34:47,250 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:34:48,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:34:48,130 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:34:49,251 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:34:51,252 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:34:53,253 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:34:55,254 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:34:57,254 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:34:59,255 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:35:01,256 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:35:03,257 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:35:03,433 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:35:03,434 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:35:05,138 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:35:05,258 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:35:07,259 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:35:09,260 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:35:11,261 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:35:13,262 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:35:15,263 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:35:17,264 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:35:18,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:35:18,727 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:35:19,265 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:35:21,266 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:35:23,267 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:35:25,268 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:35:27,269 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:35:29,270 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:35:31,271 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:35:33,272 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:35:33,951 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:35:33,951 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:35:35,217 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:35:35,273 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:35:37,274 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:35:39,275 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:35:41,276 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:35:43,277 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:35:45,278 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:35:47,279 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:35:49,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:35:49,229 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:35:49,279 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:35:51,280 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:35:53,281 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:35:55,282 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:35:57,283 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:35:59,284 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:36:01,285 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:36:03,285 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:36:04,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:36:04,541 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:36:05,286 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:36:05,298 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:36:08,287 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:36:10,288 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:36:12,289 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:36:14,289 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:36:16,290 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:36:18,291 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:36:19,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:36:19,791 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:36:20,292 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:36:22,293 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:36:24,293 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:36:26,294 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:36:28,295 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:36:30,295 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:36:32,296 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:36:34,297 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:36:35,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:36:35,054 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:36:35,378 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:36:36,298 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:36:38,298 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:36:40,299 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:36:42,300 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:36:44,300 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:36:46,301 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:36:48,302 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:36:50,303 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:36:50,353 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:36:50,354 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:36:52,303 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:36:54,304 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:36:56,305 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:36:58,306 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:37:00,306 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:37:03,307 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:37:05,308 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:37:05,459 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:37:05,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:37:05,578 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:37:07,309 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:37:09,310 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:37:11,310 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:37:13,311 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:37:15,312 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:37:17,313 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:37:19,313 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:37:20,957 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:37:20,958 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:37:21,314 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:37:23,315 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:37:25,316 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:37:27,316 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:37:29,317 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:37:31,318 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:37:33,319 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:37:35,319 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:37:35,538 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:37:36,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:37:36,248 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:37:37,320 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:37:39,321 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:37:41,322 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:37:43,322 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:37:45,323 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:37:47,324 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:37:49,325 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:37:51,325 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:37:51,475 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:37:51,476 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:37:53,326 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:37:55,327 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:37:57,327 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:37:59,328 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:38:01,329 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:38:03,330 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:38:05,330 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:38:05,618 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:38:06,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:38:06,796 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:38:07,331 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:38:09,332 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:38:11,332 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:38:13,333 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:38:15,334 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:38:17,335 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:38:19,335 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:38:21,336 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:38:22,011 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:38:22,012 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:38:23,337 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:38:25,338 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:38:27,338 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:38:29,339 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:38:31,340 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:38:34,341 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:38:35,698 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:38:36,342 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:38:37,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:38:37,376 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:38:38,342 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:38:40,343 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:38:42,344 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:38:44,345 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:38:46,345 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:38:48,346 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:38:50,347 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:38:52,347 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:38:52,600 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:38:52,601 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:38:54,348 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:38:56,349 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:38:58,350 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:39:00,350 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:39:02,351 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:39:04,352 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:39:05,779 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:39:06,353 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:39:07,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:39:07,930 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:39:08,353 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:39:10,354 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:39:12,355 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:39:14,356 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:39:16,356 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:39:18,357 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:39:20,358 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:39:22,359 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:39:23,259 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:39:23,260 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:39:24,360 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:39:26,360 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:39:28,361 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:39:30,362 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:39:32,363 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:39:34,363 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:39:35,859 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:39:36,364 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:39:38,365 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:39:38,513 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:39:38,513 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:39:40,365 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:39:42,366 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:39:45,367 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:39:47,368 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:39:49,369 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:39:51,369 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:39:53,370 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:39:53,737 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:39:53,738 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:39:55,371 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:39:57,372 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:39:59,372 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:40:01,373 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:40:03,374 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:40:05,374 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:40:05,939 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:40:07,375 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:40:09,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:40:09,044 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:40:09,376 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:40:11,377 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:40:13,377 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:40:15,378 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:40:17,379 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:40:19,379 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:40:21,380 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:40:23,381 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:40:24,349 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:40:24,349 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:40:25,382 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:40:27,383 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:40:30,384 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:40:32,384 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:40:34,385 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:40:36,020 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:40:36,386 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:40:39,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:40:39,592 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:40:54,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:40:54,815 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:41:06,099 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:41:10,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:41:10,066 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:41:25,336 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:41:25,337 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:41:36,178 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:41:40,598 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:41:40,598 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:41:50,440 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history +2022-10-25 19:41:50,446 DEBUG SenderThread:2194005 [sender.py:send():232] send: history +2022-10-25 19:41:50,496 DEBUG SenderThread:2194005 [sender.py:send():232] send: summary +2022-10-25 19:41:50,595 INFO SenderThread:2194005 [sender.py:_save_file():946] saving file wandb-summary.json with policy end +2022-10-25 19:41:51,414 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/wandb-summary.json +2022-10-25 19:41:52,415 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:41:55,844 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:41:55,846 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:41:58,418 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:42:06,289 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:42:11,070 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:42:11,070 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:42:14,425 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log +2022-10-25 19:42:26,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:42:26,294 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:42:36,373 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:42:41,520 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:42:41,521 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:42:56,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:42:56,750 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:43:06,458 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:43:11,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:43:11,979 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:43:27,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:43:27,225 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:43:36,543 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:43:42,457 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:43:42,458 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:43:57,689 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:43:57,690 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:44:06,625 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:44:12,914 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:44:12,915 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:44:28,147 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:44:28,148 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:44:36,711 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:44:43,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:44:43,377 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:44:58,606 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:44:58,606 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:45:06,798 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:45:13,833 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:45:13,833 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:45:29,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:45:29,059 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:45:36,882 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +2022-10-25 19:45:44,285 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:45:44,286 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:45:59,513 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status +2022-10-25 19:45:59,513 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status +2022-10-25 19:46:06,969 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats