diff --git "a/wandb/debug-internal.log" "b/wandb/debug-internal.log" --- "a/wandb/debug-internal.log" +++ "b/wandb/debug-internal.log" @@ -15622,3 +15622,5839 @@ 2023-11-23 09:00:53,324 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log 2023-11-23 09:00:55,762 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive 2023-11-23 09:00:57,146 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:01:01,068 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:01:01,068 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:01:02,607 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:01:03,405 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:01:03,540 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:01:06,069 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:01:08,541 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:01:11,957 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:01:13,472 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:01:14,159 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:01:16,958 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:01:19,161 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:01:22,705 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:01:23,543 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:01:24,919 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:01:27,707 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:01:29,921 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:01:33,551 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:01:33,604 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:01:33,614 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:01:35,552 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:01:39,713 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:01:40,553 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:01:43,667 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:01:44,714 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:01:45,911 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:01:50,418 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:01:50,912 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:01:53,729 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:01:55,418 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:01:56,257 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:02:00,419 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:02:01,258 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:02:03,542 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:02:05,936 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:02:06,413 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:02:07,807 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:02:10,937 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:02:11,414 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:02:16,277 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:02:16,278 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:02:16,415 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:02:17,873 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:02:21,278 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:02:21,447 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:02:26,358 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: partial_history +2023-11-23 09:02:26,360 DEBUG SenderThread:3236 [sender.py:send():380] send: history +2023-11-23 09:02:26,361 DEBUG SenderThread:3236 [sender.py:send_request():407] send_request: summary_record +2023-11-23 09:02:26,363 INFO SenderThread:3236 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-23 09:02:26,937 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/wandb-summary.json +2023-11-23 09:02:26,971 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:02:27,365 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:02:27,943 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:02:28,946 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:02:31,972 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:02:32,366 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:02:33,545 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:02:37,010 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:02:37,624 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:02:37,858 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:02:42,625 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:02:42,859 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:02:47,167 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:02:48,301 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:02:48,649 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:02:53,650 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:02:54,477 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:02:57,230 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:02:58,857 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:02:59,478 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:03:03,546 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:03:04,548 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:03:05,377 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:03:07,303 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:03:09,768 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:03:10,378 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:03:14,769 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:03:16,276 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:03:17,389 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:03:20,469 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:03:21,277 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:03:26,129 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:03:26,949 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:03:27,451 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:03:31,129 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:03:33,484 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:03:33,484 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:03:33,546 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:03:36,847 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:03:37,508 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:03:38,485 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:03:41,850 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:03:43,495 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:03:47,577 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:03:47,712 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:03:48,496 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:03:52,713 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:03:53,978 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:03:55,635 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:03:58,372 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:03:58,979 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:04:03,373 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:04:03,547 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:04:04,546 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:04:05,710 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:04:08,765 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:04:09,547 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:04:14,320 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:04:15,058 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:04:15,773 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:04:19,338 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:04:20,060 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:04:24,911 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:04:25,656 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:04:25,845 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:04:29,912 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:04:31,246 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:04:31,246 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:04:31,247 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:04:33,547 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:04:33,901 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:04:35,548 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:04:36,247 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:04:40,549 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:04:41,987 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:04:43,966 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:04:46,126 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:04:46,989 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:04:51,128 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:04:52,570 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:04:54,042 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:04:56,867 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:04:57,571 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:05:02,380 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:05:03,121 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:05:03,548 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:05:04,096 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:05:07,549 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:05:08,123 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:05:12,887 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:05:13,175 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:05:13,752 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:05:17,889 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:05:19,416 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:05:19,417 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:05:21,227 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:05:23,697 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:05:24,417 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:05:28,698 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:05:30,289 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:05:31,496 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:05:33,549 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:05:34,559 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:05:35,290 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:05:39,560 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:05:40,349 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: partial_history +2023-11-23 09:05:40,352 DEBUG SenderThread:3236 [sender.py:send():380] send: history +2023-11-23 09:05:40,352 DEBUG SenderThread:3236 [sender.py:send_request():407] send_request: summary_record +2023-11-23 09:05:40,355 INFO SenderThread:3236 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-23 09:05:40,558 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/wandb-summary.json +2023-11-23 09:05:40,958 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:05:41,564 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:05:45,357 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:05:45,969 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:05:51,012 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:05:51,259 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:05:51,623 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:05:56,017 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:05:56,260 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:06:01,261 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:06:01,826 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:06:01,838 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:06:01,838 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:06:03,549 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:06:03,711 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:06:06,557 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:06:06,838 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:06:11,559 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:06:12,417 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:06:13,775 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:06:16,635 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:06:17,418 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:06:22,178 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:06:22,910 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:06:23,836 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:06:27,178 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:06:27,911 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:06:32,930 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:06:33,479 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:06:33,549 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:06:33,991 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:06:38,481 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:06:38,551 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:06:44,033 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:06:44,045 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:06:44,196 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:06:49,197 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:06:49,629 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:06:49,630 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:06:52,123 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:06:54,631 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:06:54,833 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:06:59,834 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:07:00,012 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:07:02,204 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:07:03,550 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:07:05,076 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:07:05,551 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:07:11,007 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:07:11,209 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:07:12,269 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:07:16,008 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:07:16,209 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:07:21,330 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:07:21,526 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:07:21,773 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:07:26,775 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:07:28,023 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:07:31,395 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:07:32,256 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:07:33,024 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:07:33,551 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:07:37,552 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:07:38,686 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:07:39,446 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:07:42,834 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:07:43,687 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:07:47,836 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:07:49,449 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:07:49,450 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:07:49,520 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:07:53,656 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:07:54,450 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:07:59,514 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:07:59,589 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:08:00,327 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:08:03,552 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:08:04,553 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:08:06,697 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:08:09,651 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:08:09,949 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:08:11,698 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:08:14,950 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:08:17,196 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:08:19,774 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:08:20,412 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:08:22,197 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:08:25,414 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:08:27,684 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:08:27,832 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:08:30,960 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:08:32,685 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:08:33,553 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:08:36,479 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:08:37,904 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:08:38,216 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:08:41,480 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:08:44,732 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:08:46,976 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:08:47,980 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:08:49,733 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:08:51,977 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:08:55,625 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:08:55,861 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: partial_history +2023-11-23 09:08:55,863 DEBUG SenderThread:3236 [sender.py:send():380] send: history +2023-11-23 09:08:55,864 DEBUG SenderThread:3236 [sender.py:send_request():407] send_request: summary_record +2023-11-23 09:08:55,868 INFO SenderThread:3236 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-23 09:08:56,033 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/wandb-summary.json +2023-11-23 09:08:57,039 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:08:57,870 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:08:58,045 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:09:00,626 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:09:02,871 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:09:03,554 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:09:06,259 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:09:08,111 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:09:08,437 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:09:11,260 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:09:13,438 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:09:16,158 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:09:16,737 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:09:19,011 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:09:23,305 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:09:24,012 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:09:26,219 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:09:28,306 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:09:29,575 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:09:33,554 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:09:33,905 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:09:33,906 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:09:35,171 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:09:36,284 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:09:38,906 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:09:40,172 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:09:44,734 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:09:45,873 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:09:46,347 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:09:49,735 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:09:50,875 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:09:55,508 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:09:56,410 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:09:56,867 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:10:00,509 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:10:01,869 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:10:03,555 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:10:05,566 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:10:05,912 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:10:07,561 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:10:11,606 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:10:12,562 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:10:13,618 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:10:16,607 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:10:17,761 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:10:21,988 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:10:21,989 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:10:23,245 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:10:23,677 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:10:26,989 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:10:28,245 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:10:32,678 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:10:33,555 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:10:33,556 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:10:33,760 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:10:37,679 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:10:38,557 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:10:43,241 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:10:43,826 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:10:44,468 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:10:48,243 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:10:49,470 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:10:51,878 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:10:53,957 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:10:55,113 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:10:59,447 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:11:00,114 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:11:01,947 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:11:03,566 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:11:04,449 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:11:05,567 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:11:10,439 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:11:10,568 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:11:12,017 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:11:15,440 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:11:15,854 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:11:21,228 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:11:21,368 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:11:22,079 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:11:26,229 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:11:26,369 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:11:31,550 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:11:31,738 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:11:32,138 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:11:33,556 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:11:36,552 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:11:37,558 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:11:42,197 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:11:42,285 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:11:42,616 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:11:47,617 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:11:48,176 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:11:52,266 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:11:53,177 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:11:53,348 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:11:58,349 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:11:58,480 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:12:00,312 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:12:03,481 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:12:03,557 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:12:03,557 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:12:08,404 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: partial_history +2023-11-23 09:12:08,405 DEBUG SenderThread:3236 [sender.py:send():380] send: history +2023-11-23 09:12:08,406 DEBUG SenderThread:3236 [sender.py:send_request():407] send_request: summary_record +2023-11-23 09:12:08,408 INFO SenderThread:3236 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-23 09:12:09,206 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:12:09,206 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:12:09,379 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:12:09,380 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/wandb-summary.json +2023-11-23 09:12:09,410 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:12:10,386 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:12:14,207 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:12:14,411 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:12:19,228 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:12:19,659 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:12:20,447 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:12:24,661 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:12:26,012 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:12:28,502 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:12:30,238 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:12:31,013 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:12:33,558 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:12:35,559 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:12:36,691 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:12:36,692 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:12:38,569 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:12:40,905 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:12:41,755 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:12:45,907 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:12:47,576 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:12:48,637 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:12:50,939 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:12:52,577 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:12:55,940 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:12:58,290 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:12:58,711 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:13:01,675 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:13:03,292 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:13:03,558 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:13:07,454 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:13:07,767 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:13:08,737 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:13:12,455 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:13:14,691 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:13:17,832 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:13:17,880 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:13:19,692 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:13:22,881 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:13:24,710 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:13:27,895 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:13:28,068 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:13:29,765 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:13:33,070 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:13:33,561 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:13:35,692 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:13:37,967 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:13:38,935 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:13:40,703 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:13:43,937 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:13:46,024 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:13:46,199 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:13:49,420 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:13:51,200 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:13:54,422 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:13:56,096 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:13:56,985 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:14:00,161 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:14:03,491 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:14:03,561 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:14:05,791 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:14:06,168 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:14:08,492 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:14:10,792 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:14:13,673 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:14:16,029 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:14:16,236 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:14:18,674 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:14:21,030 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:14:23,759 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:14:23,760 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:14:26,300 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:14:26,993 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:14:28,761 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:14:31,994 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:14:33,564 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:14:34,242 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:14:36,373 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:14:37,487 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:14:39,243 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:14:42,489 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:14:44,635 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:14:46,434 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:14:47,877 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:14:49,636 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:14:52,878 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:14:54,490 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:14:55,077 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:14:58,416 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:15:01,686 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:15:03,563 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:15:03,564 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:15:04,546 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:15:06,687 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:15:08,565 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:15:12,331 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:15:13,589 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:15:14,614 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:15:17,332 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:15:18,590 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:15:22,356 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:15:22,608 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: partial_history +2023-11-23 09:15:22,610 DEBUG SenderThread:3236 [sender.py:send():380] send: history +2023-11-23 09:15:22,611 DEBUG SenderThread:3236 [sender.py:send_request():407] send_request: summary_record +2023-11-23 09:15:22,616 INFO SenderThread:3236 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-23 09:15:22,671 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/wandb-summary.json +2023-11-23 09:15:23,618 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:15:23,676 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:15:24,683 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:15:27,357 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:15:28,619 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:15:32,943 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:15:33,564 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:15:34,564 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:15:34,741 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:15:37,944 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:15:39,566 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:15:42,801 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:15:43,302 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:15:45,481 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:15:49,761 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:15:50,482 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:15:52,869 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:15:54,762 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:15:55,931 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:16:00,160 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:16:01,438 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:16:02,925 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:16:03,564 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:16:05,161 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:16:06,566 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:16:11,124 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:16:11,125 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:16:12,005 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:16:12,371 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:16:16,125 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:16:17,371 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:16:21,153 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:16:22,373 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:16:26,094 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:16:26,897 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:16:26,897 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:16:28,000 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:16:32,293 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:16:33,001 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:16:33,565 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:16:34,142 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:16:37,294 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:16:38,568 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:16:42,821 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:16:42,821 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:16:44,088 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:16:44,207 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:16:47,822 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:16:49,106 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:16:52,896 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:16:54,267 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:16:54,627 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:16:58,502 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:16:59,628 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:17:03,565 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:17:04,126 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:17:04,343 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:17:05,566 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:17:10,567 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:17:10,718 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:17:10,718 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:17:14,410 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:17:15,719 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:17:15,886 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:17:20,887 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:17:21,311 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:17:22,475 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:17:26,312 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:17:26,475 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:17:31,824 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:17:32,008 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:17:32,539 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:17:33,566 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:17:36,826 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:17:37,567 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:17:42,372 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:17:42,569 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:17:44,616 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:17:47,373 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:17:47,803 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:17:52,805 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:17:53,127 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:17:53,128 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:17:53,128 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:17:56,690 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:17:58,129 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:17:58,698 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:18:03,567 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:18:04,202 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:18:04,569 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:18:06,748 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:18:09,266 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:18:10,426 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:18:15,260 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:18:15,428 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:18:18,846 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:18:20,328 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:18:20,524 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:18:25,526 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:18:25,531 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:18:25,531 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:18:28,963 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:18:30,532 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:18:30,658 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:18:33,567 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:18:35,783 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:18:36,568 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:18:39,034 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:18:40,784 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:18:42,010 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:18:46,699 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:18:46,845 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: partial_history +2023-11-23 09:18:46,846 DEBUG SenderThread:3236 [sender.py:send():380] send: history +2023-11-23 09:18:46,847 DEBUG SenderThread:3236 [sender.py:send_request():407] send_request: summary_record +2023-11-23 09:18:46,849 INFO SenderThread:3236 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-23 09:18:47,084 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/wandb-summary.json +2023-11-23 09:18:47,851 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:18:48,089 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:18:49,093 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:18:51,700 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:18:52,852 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:18:56,807 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:18:56,807 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:18:58,086 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:18:58,151 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:19:01,808 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:19:03,087 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:19:03,568 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:19:07,102 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:19:08,225 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:19:08,661 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:19:12,104 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:19:13,662 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:19:18,011 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:19:19,486 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:19:20,306 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:19:23,115 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:19:24,487 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:19:28,954 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:19:30,125 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:19:30,378 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:19:33,568 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:19:33,980 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:19:35,569 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:19:39,104 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:19:40,466 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:19:41,449 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:19:44,105 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:19:46,451 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:19:49,163 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:19:50,528 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:19:52,445 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:19:54,164 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:19:57,447 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:19:59,241 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:20:00,597 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:20:02,567 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:20:03,569 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:20:04,243 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:20:07,570 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:20:10,127 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:20:10,664 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:20:13,320 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:20:15,128 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:20:18,321 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:20:20,610 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:20:20,741 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:20:23,906 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:20:25,612 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:20:28,908 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:20:31,018 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:20:32,809 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:20:33,569 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:20:34,570 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:20:36,019 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:20:39,571 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:20:41,854 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:20:42,924 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:20:45,129 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:20:46,856 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:20:50,130 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:20:51,903 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:20:52,991 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:20:55,326 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:20:56,904 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:21:01,086 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:21:02,850 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:21:03,056 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:21:03,581 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:21:06,582 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:21:08,775 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:21:12,189 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:21:13,120 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:21:13,776 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:21:17,190 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:21:19,250 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:21:22,613 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:21:23,180 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:21:24,251 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:21:27,614 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:21:29,325 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:21:32,786 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:21:33,245 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:21:33,570 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:21:34,328 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:21:38,572 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:21:39,527 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:21:39,528 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:21:39,528 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:21:43,307 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:21:43,865 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:21:44,529 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:21:48,867 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:21:50,450 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:21:52,368 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:21:54,836 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:21:55,451 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:21:59,837 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:22:00,764 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:22:02,437 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:22:03,572 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:22:05,573 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:22:05,765 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:22:10,574 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:22:11,127 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:22:11,559 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: partial_history +2023-11-23 09:22:11,560 DEBUG SenderThread:3236 [sender.py:send():380] send: history +2023-11-23 09:22:11,560 DEBUG SenderThread:3236 [sender.py:send_request():407] send_request: summary_record +2023-11-23 09:22:11,562 INFO SenderThread:3236 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-23 09:22:12,506 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:22:12,507 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/wandb-summary.json +2023-11-23 09:22:14,515 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:22:16,128 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:22:16,564 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:22:21,310 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:22:21,565 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:22:22,575 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:22:26,311 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:22:26,600 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:22:31,453 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:22:31,602 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:22:32,649 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:22:33,572 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:22:36,454 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:22:37,573 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:22:42,358 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:22:42,358 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:22:42,736 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:22:42,778 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:22:47,359 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:22:47,779 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:22:52,672 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:22:52,811 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:22:52,854 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:22:57,673 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:22:57,855 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:23:02,879 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:23:02,907 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:23:03,573 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:23:03,618 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:23:08,574 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:23:10,195 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:23:12,935 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:23:14,540 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:23:15,197 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:23:19,542 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:23:21,067 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:23:23,000 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:23:25,310 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:23:26,076 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:23:30,312 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:23:32,022 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:23:33,075 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:23:33,573 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:23:35,574 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:23:37,023 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:23:40,575 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:23:42,359 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:23:43,141 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:23:46,264 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:23:47,996 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:23:51,266 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:23:53,206 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:23:53,229 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:23:56,579 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:23:58,231 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:24:02,060 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:24:03,270 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:24:03,573 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:24:03,785 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:24:07,575 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:24:09,431 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:24:12,611 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:24:13,335 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:24:14,432 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:24:17,613 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:24:20,021 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:24:21,386 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:24:23,242 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:24:25,022 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:24:28,243 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:24:30,962 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:24:33,473 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:24:33,476 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:24:33,574 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:24:35,963 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:24:38,576 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:24:41,359 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:24:42,537 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:24:43,714 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:24:46,603 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:24:48,716 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:24:52,364 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:24:52,598 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:24:53,741 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:24:57,366 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:24:58,743 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:25:00,655 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:25:03,105 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:25:03,574 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:25:04,575 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:25:08,755 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:25:09,608 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:25:10,724 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:25:13,756 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:25:14,930 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:25:19,367 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:25:20,651 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:25:20,793 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:25:24,368 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:25:25,652 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:25:29,462 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:25:29,463 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:25:29,733 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: partial_history +2023-11-23 09:25:29,734 DEBUG SenderThread:3236 [sender.py:send():380] send: history +2023-11-23 09:25:29,734 DEBUG SenderThread:3236 [sender.py:send_request():407] send_request: summary_record +2023-11-23 09:25:29,737 INFO SenderThread:3236 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-23 09:25:29,851 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/wandb-summary.json +2023-11-23 09:25:30,738 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:25:30,855 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:25:32,864 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:25:33,575 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:25:34,463 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:25:36,576 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:25:40,105 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:25:40,918 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:25:42,421 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:25:45,106 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:25:47,422 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:25:50,969 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:25:51,015 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:25:53,181 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:25:57,449 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:25:58,182 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:26:01,075 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:26:02,450 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:26:03,576 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:26:03,577 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:26:08,302 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:26:08,578 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:26:11,138 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:26:13,304 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:26:13,697 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:26:18,675 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:26:18,699 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:26:21,197 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:26:23,676 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:26:23,949 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:26:28,950 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:26:28,955 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:26:28,955 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:26:31,270 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:26:33,576 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:26:33,956 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:26:34,577 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:26:39,229 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:26:40,553 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:26:41,335 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:26:44,277 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:26:45,554 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:26:49,332 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:26:50,776 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:26:51,405 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:26:54,333 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:26:55,777 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:27:00,048 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:27:00,049 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:27:01,310 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:27:01,471 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:27:03,577 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:27:05,049 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:27:06,578 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:27:10,361 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:27:10,361 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:27:11,527 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:27:11,979 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:27:15,698 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:27:16,981 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:27:20,836 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:27:21,588 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:27:22,221 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:27:25,845 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:27:27,223 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:27:31,649 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:27:31,753 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:27:32,942 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:27:33,578 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:27:37,688 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:27:38,579 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:27:41,713 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:27:42,689 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:27:44,037 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:27:48,469 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:27:49,744 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:27:50,772 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:27:53,470 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:27:54,745 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:27:58,828 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:27:59,044 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:28:00,273 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:28:03,578 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:28:04,045 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:28:05,580 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:28:08,900 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:28:09,810 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:28:11,134 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:28:14,812 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:28:16,136 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:28:18,962 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:28:20,542 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:28:21,738 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:28:26,480 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:28:26,740 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:28:29,024 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:28:31,483 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:28:32,689 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:28:33,579 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:28:37,146 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:28:38,325 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:28:39,099 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:28:42,147 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:28:43,326 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:28:47,050 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: partial_history +2023-11-23 09:28:47,051 DEBUG SenderThread:3236 [sender.py:send():380] send: history +2023-11-23 09:28:47,052 DEBUG SenderThread:3236 [sender.py:send_request():407] send_request: summary_record +2023-11-23 09:28:47,055 INFO SenderThread:3236 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-23 09:28:47,157 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/wandb-summary.json +2023-11-23 09:28:47,841 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:28:49,056 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:28:49,170 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:28:52,842 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:28:54,058 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:28:57,228 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:28:58,404 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:28:59,594 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:29:03,406 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:29:03,580 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:29:05,582 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:29:07,288 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:29:09,241 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:29:10,593 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:29:15,308 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:29:15,595 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:29:17,386 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:29:20,309 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:29:21,592 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:29:25,688 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:29:26,593 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:29:29,464 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:29:30,689 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:29:31,843 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:29:33,582 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:29:36,234 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:29:37,422 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:29:37,524 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:29:41,235 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:29:42,423 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:29:46,889 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:29:47,588 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:29:48,096 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:29:51,890 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:29:53,097 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:29:57,296 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:29:57,666 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:29:58,820 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:30:02,297 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:30:03,583 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:30:04,585 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:30:07,736 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:30:08,075 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:30:10,303 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:30:13,966 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:30:13,967 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:30:16,195 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:30:17,797 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:30:18,967 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:30:21,195 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:30:24,786 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:30:27,125 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:30:27,858 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:30:29,787 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:30:32,126 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:30:33,596 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:30:35,688 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:30:36,923 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:30:37,915 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:30:40,689 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:30:42,917 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:30:45,761 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:30:45,762 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:30:46,984 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:30:48,165 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:30:50,770 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:30:53,166 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:30:56,482 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:30:57,050 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:30:59,051 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:31:01,708 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:31:03,584 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:31:04,585 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:31:08,023 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:31:10,372 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:31:11,129 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:31:13,024 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:31:15,372 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:31:18,155 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:31:20,437 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:31:21,277 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:31:23,156 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:31:25,438 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:31:28,885 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:31:28,886 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:31:30,439 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:31:33,584 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:31:33,675 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:31:33,887 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:31:35,585 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:31:38,887 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:31:40,587 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:31:43,743 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:31:44,056 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:31:46,159 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:31:49,058 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:31:51,160 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:31:53,816 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:31:54,744 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:31:56,944 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:32:00,153 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:32:02,242 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:32:03,585 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:32:03,882 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:32:05,154 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:32:07,586 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:32:10,684 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:32:10,957 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: partial_history +2023-11-23 09:32:10,958 DEBUG SenderThread:3236 [sender.py:send():380] send: history +2023-11-23 09:32:10,959 DEBUG SenderThread:3236 [sender.py:send_request():407] send_request: summary_record +2023-11-23 09:32:10,961 INFO SenderThread:3236 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-23 09:32:11,941 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:32:11,942 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/wandb-summary.json +2023-11-23 09:32:12,945 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:32:12,963 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:32:15,686 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:32:17,964 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:32:21,192 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:32:23,478 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:32:24,016 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:32:26,194 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:32:28,480 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:32:31,895 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:32:33,082 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:32:33,585 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:32:33,586 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:32:36,896 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:32:38,588 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:32:42,850 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:32:43,156 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:32:44,207 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:32:47,852 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:32:49,209 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:32:51,205 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:32:53,669 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:32:54,914 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:32:59,163 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:32:59,163 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:32:59,915 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:33:01,277 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:33:03,586 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:33:04,164 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:33:05,587 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:33:10,106 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:33:11,338 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:33:11,502 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:33:15,107 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:33:16,503 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:33:21,088 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:33:21,401 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:33:22,342 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:33:26,089 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:33:27,343 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:33:31,430 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:33:31,473 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:33:32,593 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:33:33,586 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:33:36,461 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:33:38,588 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:33:41,537 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:33:42,260 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:33:44,503 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:33:47,261 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:33:49,504 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:33:51,598 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:33:52,826 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:33:55,226 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:33:58,652 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:34:00,828 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:34:01,674 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:34:03,590 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:34:03,653 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:34:06,591 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:34:09,139 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:34:11,739 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:34:12,441 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:34:14,140 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:34:17,443 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:34:20,117 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:34:21,808 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:34:23,376 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:34:25,118 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:34:28,378 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:34:30,431 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:34:31,869 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:34:33,587 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:34:33,588 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:34:35,876 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:34:38,590 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:34:41,727 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:34:41,939 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:34:43,875 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:34:46,728 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:34:48,876 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:34:52,418 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:34:53,884 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:34:57,419 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:34:58,885 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:35:03,235 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:35:03,588 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:35:04,055 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:35:04,589 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:35:08,236 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:35:09,590 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:35:13,908 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:35:14,158 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:35:15,459 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:35:18,909 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:35:20,461 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:35:24,229 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:35:24,768 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:35:26,023 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:35:30,642 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:35:30,643 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:35:30,643 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:35:31,025 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:35:33,283 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:35:33,589 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:35:35,644 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:35:36,590 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:35:40,644 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:35:41,591 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:35:45,646 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:35:46,591 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:35:51,510 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:35:51,593 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:35:54,018 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: partial_history +2023-11-23 09:35:54,020 DEBUG SenderThread:3236 [sender.py:send():380] send: history +2023-11-23 09:35:54,021 DEBUG SenderThread:3236 [sender.py:send_request():407] send_request: summary_record +2023-11-23 09:35:54,024 INFO SenderThread:3236 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-23 09:35:54,427 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/wandb-summary.json +2023-11-23 09:35:55,432 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:35:56,577 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:35:57,026 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:36:02,027 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:36:02,527 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:36:03,590 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:36:05,496 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:36:07,528 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:36:07,984 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:36:12,967 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:36:12,985 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:36:15,584 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:36:17,968 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:36:18,292 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:36:23,294 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:36:23,965 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:36:25,648 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:36:28,966 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:36:29,063 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:36:33,590 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:36:34,283 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:36:34,284 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:36:34,591 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:36:35,714 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:36:39,284 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:36:39,592 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:36:44,922 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:36:45,088 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:36:45,776 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:36:50,090 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:36:51,271 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:36:53,833 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:36:55,577 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:36:56,272 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:37:00,579 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:37:02,123 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:37:02,124 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:37:03,591 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:37:03,907 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:37:05,592 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:37:07,125 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:37:10,593 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:37:12,493 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:37:13,968 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:37:15,665 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:37:17,494 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:37:20,667 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:37:22,845 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:37:24,046 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:37:26,120 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:37:27,845 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:37:31,122 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:37:33,124 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:37:33,591 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:37:34,110 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:37:36,592 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:37:40,044 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:37:42,280 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:37:44,178 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:37:45,045 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:37:47,281 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:37:50,621 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:37:50,622 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:37:52,234 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:37:52,785 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:37:55,623 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:37:57,786 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:38:01,137 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:38:01,162 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:38:02,295 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:38:03,592 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:38:03,594 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:38:06,508 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:38:08,594 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:38:11,524 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:38:12,359 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:38:13,801 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:38:16,525 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:38:18,803 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:38:22,200 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:38:22,427 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:38:24,366 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:38:27,202 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:38:29,368 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:38:31,490 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:38:33,080 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:38:33,593 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:38:34,593 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:38:38,507 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:38:39,595 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:38:41,561 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:38:43,508 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:38:44,682 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:38:49,336 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:38:49,336 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:38:50,630 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:38:51,624 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:38:54,337 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:38:55,631 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:39:00,198 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:39:01,592 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:39:01,685 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:39:03,593 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:39:05,199 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:39:06,594 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:39:09,227 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: partial_history +2023-11-23 09:39:09,228 DEBUG SenderThread:3236 [sender.py:send():380] send: history +2023-11-23 09:39:09,229 DEBUG SenderThread:3236 [sender.py:send_request():407] send_request: summary_record +2023-11-23 09:39:09,232 INFO SenderThread:3236 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-23 09:39:09,739 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/wandb-summary.json +2023-11-23 09:39:11,055 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:39:11,749 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:39:12,234 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:39:17,237 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:39:17,435 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:39:17,436 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:39:19,813 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:39:22,437 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:39:22,792 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:39:27,793 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:39:28,100 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:39:29,883 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:39:33,101 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:39:33,353 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:39:33,594 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:39:38,284 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:39:38,419 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:39:39,949 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:39:43,285 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:39:43,420 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:39:48,436 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:39:49,060 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:39:50,020 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:39:53,669 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:39:54,061 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:39:59,120 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:39:59,526 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:40:00,085 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:40:03,604 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:40:04,121 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:40:04,605 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:40:09,644 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:40:09,909 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:40:10,165 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:40:14,645 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:40:14,910 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:40:19,961 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:40:20,113 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:40:20,228 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:40:25,114 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:40:26,386 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:40:30,287 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:40:30,611 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:40:31,388 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:40:33,594 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:40:36,595 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:40:37,281 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:40:37,282 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:40:40,387 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:40:42,009 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:40:42,282 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:40:47,011 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:40:47,300 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:40:50,467 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:40:52,301 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:40:52,672 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:40:57,674 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:40:58,075 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:41:00,535 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:41:03,076 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:41:03,231 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:41:03,595 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:41:08,471 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:41:08,720 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:41:10,603 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:41:13,471 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:41:13,721 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:41:18,883 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:41:19,290 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:41:20,667 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:41:23,884 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:41:24,291 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:41:29,002 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:41:29,470 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:41:29,731 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:41:33,596 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:41:34,003 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:41:34,597 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:41:39,794 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:41:39,854 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:41:40,360 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:41:45,361 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:41:45,954 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:41:47,847 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:41:50,955 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:41:51,181 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:41:56,183 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:41:56,416 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:41:57,908 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:42:01,417 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:42:01,786 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:42:03,597 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:42:06,974 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:42:07,114 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:42:07,977 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:42:11,975 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:42:12,115 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:42:17,382 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:42:17,685 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:42:18,043 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:42:22,384 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:42:22,687 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:42:25,384 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: partial_history +2023-11-23 09:42:25,386 DEBUG SenderThread:3236 [sender.py:send():380] send: history +2023-11-23 09:42:25,387 DEBUG SenderThread:3236 [sender.py:send_request():407] send_request: summary_record +2023-11-23 09:42:25,389 INFO SenderThread:3236 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-23 09:42:26,092 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/wandb-summary.json +2023-11-23 09:42:27,097 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:42:28,101 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:42:28,195 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:42:28,391 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:42:33,393 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:42:33,598 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:42:33,686 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:42:33,687 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:42:33,687 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:42:36,157 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:42:38,688 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:42:39,163 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:42:44,164 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:42:44,401 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:42:46,228 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:42:49,402 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:42:49,628 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:42:54,741 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:42:54,903 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:42:56,283 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:42:59,742 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:42:59,904 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:43:03,598 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:43:05,182 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:43:05,687 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:43:06,341 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:43:10,183 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:43:10,715 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:43:15,827 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:43:16,405 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:43:16,507 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:43:20,828 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:43:21,917 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:43:21,917 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:43:24,455 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:43:26,130 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:43:26,918 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:43:31,131 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:43:32,363 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:43:33,599 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:43:34,526 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:43:36,600 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:43:37,364 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:43:41,601 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:43:42,920 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:43:44,593 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:43:47,158 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:43:47,921 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:43:52,776 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:43:53,499 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:43:53,499 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:43:54,664 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:43:57,777 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:43:58,499 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:44:03,072 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:44:03,599 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:44:03,688 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:44:04,725 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:44:08,601 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:44:10,203 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:44:13,974 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:44:14,789 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:44:15,203 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:44:18,976 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:44:20,527 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:44:24,720 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:44:24,865 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:44:25,528 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:44:29,721 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:44:30,913 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:44:32,919 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:44:33,600 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:44:35,601 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:44:35,914 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:44:40,602 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:44:40,991 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:44:41,979 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:44:46,007 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:44:46,194 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:44:51,662 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:44:51,663 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:44:52,012 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:44:52,042 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:44:56,663 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:44:57,013 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:45:02,101 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:45:02,258 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:45:02,532 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:45:03,601 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:45:07,603 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:45:08,768 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:45:12,187 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:45:13,020 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:45:13,772 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:45:18,022 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:45:19,462 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:45:20,242 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:45:23,589 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:45:24,467 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:45:28,590 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:45:30,213 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:45:30,318 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:45:33,621 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:45:33,623 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:45:35,214 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:45:38,623 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:45:39,290 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: partial_history +2023-11-23 09:45:39,291 DEBUG SenderThread:3236 [sender.py:send():380] send: history +2023-11-23 09:45:39,292 DEBUG SenderThread:3236 [sender.py:send_request():407] send_request: summary_record +2023-11-23 09:45:39,294 INFO SenderThread:3236 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-23 09:45:39,390 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/wandb-summary.json +2023-11-23 09:45:40,396 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:45:41,086 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:45:41,399 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:45:44,297 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:45:46,088 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:45:49,298 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:45:50,460 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:45:51,679 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:45:54,427 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:45:57,896 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:46:00,115 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:46:00,520 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:46:02,897 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:46:03,621 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:46:05,622 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:46:07,898 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:46:10,623 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:46:12,904 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:46:14,603 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:46:16,060 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:46:17,905 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:46:21,062 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:46:23,780 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:46:23,781 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:46:24,736 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:46:27,057 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:46:28,781 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:46:32,058 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:46:33,621 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:46:34,534 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:46:34,809 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:46:37,622 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:46:41,046 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:46:43,278 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:46:44,876 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:46:46,047 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:46:48,279 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:46:51,541 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:46:51,541 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:46:52,941 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:46:53,804 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:46:56,542 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:46:58,806 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:47:02,105 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:47:03,008 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:47:03,622 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:47:04,622 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:47:07,106 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:47:09,624 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:47:12,606 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:47:13,077 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:47:14,788 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:47:17,607 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:47:19,790 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:47:22,377 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:47:23,541 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:47:24,830 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:47:29,218 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:47:29,832 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:47:32,528 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:47:33,622 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:47:34,219 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:47:35,623 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:47:39,499 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:47:39,499 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:47:40,580 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:47:40,684 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:47:44,500 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:47:45,685 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:47:49,990 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:47:50,649 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:47:51,220 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:47:54,991 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:47:56,222 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:48:00,681 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:48:00,714 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:48:01,885 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:48:03,623 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:48:05,682 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:48:07,624 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:48:10,783 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:48:11,280 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:48:13,622 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:48:17,127 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:48:19,301 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:48:20,838 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:48:22,128 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:48:24,302 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:48:27,622 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:48:29,982 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:48:30,908 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:48:32,622 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:48:33,623 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:48:35,625 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:48:38,079 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:48:38,079 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:48:38,960 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:48:41,457 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:48:43,109 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:48:46,459 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:48:49,033 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:48:49,064 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:48:51,652 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:48:54,034 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:48:56,654 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:48:57,366 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: partial_history +2023-11-23 09:48:57,368 DEBUG SenderThread:3236 [sender.py:send():380] send: history +2023-11-23 09:48:57,368 DEBUG SenderThread:3236 [sender.py:send_request():407] send_request: summary_record +2023-11-23 09:48:57,371 INFO SenderThread:3236 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-23 09:48:58,129 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/wandb-summary.json +2023-11-23 09:48:59,060 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:48:59,135 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:49:00,139 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:49:02,373 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:49:03,624 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:49:04,062 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:49:08,217 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:49:09,079 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:49:09,205 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:49:13,218 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:49:15,507 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:49:18,271 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:49:18,748 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:49:20,509 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:49:23,749 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:49:26,137 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:49:28,335 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:49:29,208 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:49:31,138 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:49:33,626 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:49:34,628 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:49:36,261 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:49:36,392 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:49:40,459 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:49:41,270 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:49:46,419 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:49:46,456 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:49:46,839 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:49:51,420 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:49:51,840 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:49:56,517 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:49:57,000 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:49:57,195 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:50:02,196 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:50:03,539 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:50:03,650 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:50:06,581 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:50:07,916 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:50:08,541 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:50:12,917 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:50:14,093 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:50:16,647 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:50:18,317 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:50:19,094 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:50:23,320 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:50:24,964 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:50:24,964 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:50:26,710 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:50:29,216 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:50:29,964 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:50:33,627 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:50:34,764 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:50:34,783 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:50:35,425 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:50:39,783 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:50:40,438 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:50:44,835 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:50:44,870 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:50:45,620 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:50:49,871 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:50:50,640 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:50:54,913 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:50:54,914 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:50:56,553 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:50:56,554 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:50:59,915 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:51:02,496 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:51:03,628 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:51:04,995 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:51:05,676 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:51:07,497 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:51:10,677 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:51:12,498 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:51:15,678 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:51:17,499 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:51:20,679 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:51:22,499 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:51:25,680 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:51:27,500 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:51:30,680 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:51:33,383 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:51:33,629 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:51:36,300 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:51:37,168 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:51:38,384 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:51:41,301 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:51:43,413 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:51:46,524 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:51:47,240 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:51:48,414 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:51:51,524 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:51:54,280 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:51:54,281 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:51:56,621 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:51:57,309 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:51:59,300 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:52:01,622 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:52:03,630 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:52:04,338 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:52:07,377 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:52:07,505 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:52:09,339 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:52:12,507 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:52:14,626 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:52:17,434 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:52:17,947 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:52:19,627 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:52:22,949 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:52:25,401 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:52:26,496 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:52:28,742 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:52:30,402 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:52:33,631 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:52:34,124 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:52:34,125 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: partial_history +2023-11-23 09:52:34,126 DEBUG SenderThread:3236 [sender.py:send():380] send: history +2023-11-23 09:52:34,126 DEBUG SenderThread:3236 [sender.py:send_request():407] send_request: summary_record +2023-11-23 09:52:34,129 INFO SenderThread:3236 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-23 09:52:34,545 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:52:34,546 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/wandb-summary.json +2023-11-23 09:52:35,920 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:52:36,554 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:52:39,131 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:52:40,922 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:52:44,605 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:52:44,722 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:52:46,601 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:52:49,723 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:52:51,884 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:52:54,672 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:52:55,118 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:52:56,885 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:53:00,119 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:53:02,527 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:53:03,631 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:53:04,746 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:53:05,632 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:53:07,528 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:53:10,634 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:53:13,292 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:53:14,810 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:53:16,444 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:53:18,293 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:53:21,445 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:53:22,868 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:53:23,401 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:53:26,735 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:53:28,515 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:53:32,454 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:53:32,935 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:53:33,632 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:53:34,103 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:53:37,633 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:53:40,486 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:53:40,487 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:53:42,977 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:53:42,995 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:53:45,488 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:53:47,978 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:53:50,663 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:53:53,106 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:53:53,956 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:53:55,665 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:53:58,957 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:54:01,474 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:54:03,163 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:54:03,632 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:54:04,633 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:54:06,475 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:54:09,634 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:54:12,088 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:54:13,230 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:54:15,443 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:54:17,106 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:54:20,445 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:54:22,996 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:54:23,294 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:54:26,336 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:54:27,997 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:54:31,864 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:54:33,358 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:54:33,633 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:54:33,641 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:54:37,635 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:54:39,499 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:54:42,734 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:54:43,583 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:54:44,500 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:54:47,735 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:54:50,079 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:54:51,631 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:54:53,337 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:54:55,079 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:54:58,339 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:55:00,562 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:55:00,694 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:55:03,633 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:55:03,634 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:55:05,563 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:55:08,635 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:55:10,756 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:55:11,088 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:55:14,373 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:55:17,441 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:55:19,594 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:55:20,826 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:55:22,442 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:55:24,595 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:55:28,185 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:55:28,185 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:55:28,899 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:55:30,396 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:55:33,186 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:55:33,634 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:55:35,635 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:55:38,782 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:55:38,969 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:55:40,937 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:55:43,783 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:55:45,938 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:55:47,514 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: partial_history +2023-11-23 09:55:47,516 DEBUG SenderThread:3236 [sender.py:send():380] send: history +2023-11-23 09:55:47,516 DEBUG SenderThread:3236 [sender.py:send_request():407] send_request: summary_record +2023-11-23 09:55:47,519 INFO SenderThread:3236 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-23 09:55:48,032 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:55:48,033 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/wandb-summary.json +2023-11-23 09:55:49,041 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:55:49,284 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:55:51,521 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:55:54,286 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:55:56,522 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:55:59,121 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:55:59,937 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:56:02,174 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:56:03,634 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:56:05,709 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:56:07,858 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:56:09,192 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:56:10,710 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:56:12,859 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:56:15,994 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:56:17,259 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:56:18,166 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:56:20,995 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:56:23,167 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:56:26,937 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:56:27,330 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:56:28,206 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:56:31,956 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:56:33,208 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:56:33,636 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:56:37,400 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:56:37,510 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:56:38,778 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:56:43,780 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:56:43,993 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:56:43,994 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:56:47,459 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:56:48,994 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:56:49,147 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:56:54,149 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:56:54,298 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:56:55,595 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:56:59,299 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:56:59,581 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:57:03,636 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:57:04,729 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:57:04,988 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:57:05,665 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:57:09,730 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:57:09,989 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:57:15,703 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:57:15,735 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:57:15,959 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:57:20,704 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:57:20,960 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:57:25,800 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:57:25,937 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:57:25,937 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:57:26,522 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:57:31,524 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:57:31,707 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:57:33,637 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:57:34,880 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:57:36,638 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:57:36,708 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:57:41,639 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:57:42,127 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:57:42,127 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:57:42,937 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:57:47,128 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:57:47,375 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:57:52,525 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:57:52,740 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:57:53,008 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:57:57,526 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:57:57,740 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:58:02,985 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:58:03,084 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:58:03,266 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:58:03,637 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:58:07,988 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:58:08,639 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:58:13,127 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:58:13,159 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:58:14,203 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:58:19,234 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:58:19,380 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:58:21,213 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:58:24,382 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:58:24,558 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:58:29,559 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:58:29,865 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:58:29,866 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:58:31,277 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:58:33,641 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:58:34,641 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:58:34,867 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:58:39,643 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:58:40,739 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:58:41,345 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:58:44,914 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:58:45,740 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:58:50,498 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:58:51,259 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:58:51,415 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:58:55,499 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:58:56,261 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:58:59,094 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: partial_history +2023-11-23 09:58:59,096 DEBUG SenderThread:3236 [sender.py:send():380] send: history +2023-11-23 09:58:59,096 DEBUG SenderThread:3236 [sender.py:send_request():407] send_request: summary_record +2023-11-23 09:58:59,099 INFO SenderThread:3236 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-23 09:58:59,470 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/wandb-summary.json +2023-11-23 09:59:00,475 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:59:01,100 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:59:01,480 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:59:01,917 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:59:03,638 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:59:06,639 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:59:07,679 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:59:09,536 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:59:11,877 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:59:12,680 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:59:16,878 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:59:18,353 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:59:19,607 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:59:22,468 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:59:23,354 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:59:27,469 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:59:28,717 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:59:29,675 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:59:33,010 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:59:33,649 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 09:59:33,718 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:59:38,502 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:59:39,249 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:59:39,745 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:59:43,503 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:59:44,251 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:59:49,365 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:59:49,805 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 09:59:50,211 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:59:54,366 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:59:56,337 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 09:59:59,547 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 09:59:59,871 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:00:01,339 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:00:03,639 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:00:04,640 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:00:06,870 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:00:09,936 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:00:10,279 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:00:11,871 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:00:15,280 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:00:17,575 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:00:17,991 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:00:20,817 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:00:22,576 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:00:25,819 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:00:27,046 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:00:27,717 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:00:31,280 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:00:33,037 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:00:33,640 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:00:36,641 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:00:39,191 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:00:41,141 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:00:41,654 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:00:44,193 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:00:46,655 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:00:49,694 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:00:51,209 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:00:51,832 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:00:54,695 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:00:56,833 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:01:00,257 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:01:00,258 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:01:01,268 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:01:02,487 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:01:03,641 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:01:05,259 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:01:07,643 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:01:10,713 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:01:11,338 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:01:12,973 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:01:15,714 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:01:17,974 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:01:21,191 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:01:21,402 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:01:23,435 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:01:26,193 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:01:28,436 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:01:29,464 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:01:31,875 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:01:33,641 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:01:33,643 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:01:37,634 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:01:38,644 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:01:39,565 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:01:42,635 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:01:43,899 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:01:48,414 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:01:49,571 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:01:49,639 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:01:53,415 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:01:54,572 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:01:58,875 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:01:59,705 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:02:00,181 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:02:03,642 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:02:03,957 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:02:05,643 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:02:09,441 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:02:09,772 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:02:10,702 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:02:14,443 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:02:15,704 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:02:17,702 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: partial_history +2023-11-23 10:02:17,703 DEBUG SenderThread:3236 [sender.py:send():380] send: history +2023-11-23 10:02:17,703 DEBUG SenderThread:3236 [sender.py:send_request():407] send_request: summary_record +2023-11-23 10:02:17,706 INFO SenderThread:3236 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-23 10:02:17,833 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/wandb-summary.json +2023-11-23 10:02:18,837 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:02:19,505 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:02:19,842 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:02:20,708 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:02:25,710 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:02:25,909 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:02:29,902 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:02:30,910 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:02:31,149 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:02:33,643 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:02:36,243 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:02:36,421 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:02:37,955 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:02:41,254 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:02:41,421 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:02:47,094 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:02:47,332 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:02:48,016 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:02:52,095 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:02:52,333 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:02:57,504 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:02:57,721 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:02:58,073 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:03:02,723 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:03:03,646 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:03:03,771 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:03:06,143 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:03:07,961 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:03:08,772 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:03:12,963 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:03:14,299 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:03:14,300 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:03:16,211 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:03:18,579 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:03:19,300 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:03:23,580 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:03:24,633 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:03:26,264 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:03:28,814 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:03:29,634 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:03:33,645 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:03:34,646 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:03:35,063 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:03:36,331 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:03:39,647 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:03:40,064 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:03:45,092 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:03:45,340 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:03:45,396 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:03:50,341 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:03:51,553 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:03:53,447 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:03:55,763 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:03:56,554 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:04:00,764 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:04:01,555 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:04:03,646 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:04:06,556 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:04:06,647 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:04:11,647 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:04:12,085 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:04:16,648 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:04:18,051 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:04:18,052 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:04:21,807 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:04:22,188 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:04:23,053 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:04:27,190 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:04:28,540 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:04:28,541 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:04:31,946 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:04:32,869 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:04:33,542 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:04:33,646 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:04:38,648 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:04:39,066 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:04:42,004 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:04:44,067 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:04:44,351 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:04:49,999 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:04:50,349 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:04:52,099 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:04:55,000 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:04:55,350 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:05:00,186 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:05:00,385 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:05:00,529 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:05:03,647 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:05:05,648 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:05:06,851 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:05:09,503 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:05:11,110 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:05:11,853 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:05:16,112 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:05:17,257 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:05:17,257 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:05:19,568 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:05:21,622 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:05:22,258 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:05:26,623 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:05:27,619 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:05:28,040 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:05:32,197 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:05:33,041 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:05:33,650 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:05:37,686 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:05:38,017 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:05:38,538 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:05:43,018 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:05:43,540 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:05:46,513 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: partial_history +2023-11-23 10:05:46,515 DEBUG SenderThread:3236 [sender.py:send():380] send: history +2023-11-23 10:05:46,515 DEBUG SenderThread:3236 [sender.py:send_request():407] send_request: summary_record +2023-11-23 10:05:46,517 INFO SenderThread:3236 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-23 10:05:46,756 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:05:46,757 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/wandb-summary.json +2023-11-23 10:05:47,761 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:05:48,519 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:05:49,347 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:05:53,520 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:05:54,573 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:05:57,832 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:05:58,745 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:05:59,574 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:06:03,650 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:06:04,653 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:06:05,306 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:06:07,905 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:06:09,678 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:06:10,307 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:06:14,679 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:06:15,962 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:06:15,988 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:06:20,274 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:06:20,990 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:06:25,685 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:06:26,025 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:06:26,626 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:06:30,686 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:06:31,627 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:06:33,651 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:06:36,084 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:06:36,260 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:06:36,822 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:06:41,261 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:06:41,823 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:06:46,155 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:06:46,989 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:06:47,029 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:06:52,030 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:06:53,495 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:06:56,214 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:06:57,972 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:06:58,497 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:07:02,974 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:07:03,499 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:07:03,499 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:07:03,652 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:07:06,272 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:07:08,500 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:07:08,722 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:07:13,724 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:07:13,989 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:07:13,990 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:07:16,346 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:07:19,050 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:07:19,319 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:07:24,292 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:07:24,321 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:07:26,422 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:07:29,293 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:07:29,388 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:07:33,654 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:07:34,731 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:07:34,922 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:07:36,485 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:07:39,732 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:07:39,923 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:07:44,594 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:07:44,990 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:07:45,350 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:07:50,351 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:07:51,702 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:07:53,651 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:07:56,050 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:07:56,703 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:08:01,051 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:08:02,322 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:08:02,322 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:08:03,654 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:08:03,716 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:08:06,656 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:08:07,350 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:08:11,658 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:08:12,383 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:08:13,785 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:08:16,767 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:08:17,384 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:08:21,769 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:08:22,624 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:08:23,891 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:08:26,939 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:08:27,625 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:08:31,941 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:08:32,702 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:08:32,702 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:08:33,655 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:08:33,966 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:08:37,656 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:08:37,703 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:08:42,658 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:08:42,768 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:08:44,042 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:08:47,769 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:08:48,176 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:08:53,313 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:08:53,568 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:08:54,107 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:08:58,314 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:08:58,569 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:09:02,392 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: partial_history +2023-11-23 10:09:02,393 DEBUG SenderThread:3236 [sender.py:send():380] send: history +2023-11-23 10:09:02,394 DEBUG SenderThread:3236 [sender.py:send_request():407] send_request: summary_record +2023-11-23 10:09:02,397 INFO SenderThread:3236 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-23 10:09:03,176 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:09:03,177 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/wandb-summary.json +2023-11-23 10:09:03,655 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:09:03,656 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:09:03,671 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:09:04,181 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:09:08,657 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:09:08,672 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:09:13,949 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:09:14,224 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:09:14,251 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:09:19,226 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:09:20,370 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:09:24,317 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:09:24,599 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:09:25,369 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:09:29,601 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:09:30,470 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:09:30,471 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:09:33,655 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:09:34,383 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:09:34,656 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:09:35,472 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:09:39,658 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:09:40,820 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:09:42,445 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:09:45,227 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:09:45,821 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:09:50,228 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:09:51,309 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:09:52,536 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:09:55,489 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:09:56,310 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:10:00,774 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:10:01,471 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:10:02,600 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:10:03,656 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:10:06,657 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:10:08,033 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:10:10,650 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:10:12,231 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:10:13,034 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:10:17,233 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:10:18,733 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:10:18,733 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:10:20,721 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:10:22,896 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:10:23,734 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:10:27,898 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:10:29,115 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:10:30,783 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:10:33,270 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:10:33,657 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:10:34,116 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:10:38,950 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:10:39,395 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:10:39,960 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:10:43,950 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:10:44,397 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:10:48,018 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:10:49,245 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:10:50,097 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:10:54,247 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:10:56,048 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:10:59,586 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:11:00,107 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:11:01,049 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:11:03,657 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:11:04,658 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:11:07,002 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:11:08,163 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:11:10,257 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:11:12,003 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:11:15,259 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:11:17,097 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:11:18,224 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:11:20,669 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:11:22,415 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:11:25,671 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:11:28,009 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:11:28,291 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:11:31,177 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:11:33,659 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:11:34,254 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:11:34,254 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:11:36,416 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:11:36,566 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:11:39,256 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:11:41,417 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:11:44,824 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:11:46,636 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:11:47,183 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:11:49,825 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:11:52,184 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:11:55,671 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:11:56,707 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:11:57,885 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:12:00,672 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:12:02,887 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:12:03,660 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:12:06,192 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:12:06,778 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:12:08,551 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:12:12,662 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:12:13,552 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:12:13,992 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: partial_history +2023-11-23 10:12:13,993 DEBUG SenderThread:3236 [sender.py:send():380] send: history +2023-11-23 10:12:13,994 DEBUG SenderThread:3236 [sender.py:send_request():407] send_request: summary_record +2023-11-23 10:12:13,996 INFO SenderThread:3236 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-23 10:12:14,840 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/wandb-summary.json +2023-11-23 10:12:15,847 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:12:16,851 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:12:17,663 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:12:18,998 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:12:23,428 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:12:24,873 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:12:26,918 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:12:28,429 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:12:29,873 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:12:33,660 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:12:34,002 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:12:34,083 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:12:35,661 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:12:39,084 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:12:40,662 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:12:44,069 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:12:44,946 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:12:46,211 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:12:51,213 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:12:51,445 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:12:51,446 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:12:54,135 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:12:56,447 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:12:56,730 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:13:01,732 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:13:01,779 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:13:03,660 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:13:04,205 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:13:06,780 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:13:07,661 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:13:11,954 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:13:12,264 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:13:13,112 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:13:16,955 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:13:18,113 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:13:22,218 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:13:22,339 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:13:23,447 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:13:27,219 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:13:28,448 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:13:32,406 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:13:32,742 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:13:33,661 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:13:33,662 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:13:38,663 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:13:39,194 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:13:42,471 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:13:44,195 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:13:44,455 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:13:49,457 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:13:49,545 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:13:49,546 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:13:50,518 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:13:54,546 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:13:54,689 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:13:59,802 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:13:59,954 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:14:00,785 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:14:03,661 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:14:04,818 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:14:05,663 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:14:10,114 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:14:10,324 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:14:11,566 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:14:15,326 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:14:16,567 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:14:20,184 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:14:20,545 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:14:22,337 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:14:26,864 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:14:27,339 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:14:30,255 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:14:31,866 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:14:33,279 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:14:33,662 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:14:37,545 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:14:37,546 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:14:38,829 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:14:40,314 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:14:42,546 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:14:43,830 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:14:47,958 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:14:48,373 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:14:49,170 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:14:52,959 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:14:54,172 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:14:58,449 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:14:58,512 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:14:59,777 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:15:03,513 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:15:03,662 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:15:05,663 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:15:09,726 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:15:10,664 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:15:12,530 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:15:14,728 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:15:15,947 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:15:19,802 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:15:19,802 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:15:20,949 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:15:22,595 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:15:24,803 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:15:25,974 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:15:30,206 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:15:30,740 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: partial_history +2023-11-23 10:15:30,742 DEBUG SenderThread:3236 [sender.py:send():380] send: history +2023-11-23 10:15:30,742 DEBUG SenderThread:3236 [sender.py:send_request():407] send_request: summary_record +2023-11-23 10:15:30,745 INFO SenderThread:3236 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-23 10:15:31,660 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:15:31,661 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/wandb-summary.json +2023-11-23 10:15:31,747 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:15:32,668 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:15:33,663 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:15:35,217 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:15:37,665 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:15:41,159 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:15:42,734 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:15:43,359 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:15:46,160 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:15:48,361 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:15:51,802 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:15:52,789 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:15:54,044 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:15:58,314 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:15:59,045 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:16:00,839 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:16:03,325 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:16:03,664 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:16:04,665 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:16:09,025 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:16:09,026 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:16:10,163 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:16:10,897 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:16:14,026 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:16:15,165 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:16:19,223 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:16:20,498 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:16:20,952 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:16:24,224 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:16:25,500 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:16:29,730 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:16:31,022 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:16:31,079 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:16:33,664 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:16:34,731 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:16:36,678 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:16:39,075 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:16:40,373 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:16:42,582 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:16:45,917 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:16:48,327 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:16:49,140 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:16:50,918 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:16:53,328 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:16:56,858 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:16:59,082 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:16:59,205 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:17:01,865 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:17:03,665 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:17:04,668 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:17:07,309 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:17:08,284 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:17:10,245 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:17:12,310 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:17:15,247 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:17:17,674 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:17:18,343 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:17:20,844 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:17:22,675 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:17:25,846 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:17:27,945 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:17:28,417 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:17:31,184 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:17:33,667 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:17:34,453 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:17:36,471 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:17:36,627 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:17:39,455 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:17:41,628 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:17:44,778 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:17:46,530 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:17:47,039 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:17:49,780 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:17:52,040 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:17:55,518 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:17:56,603 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:17:57,865 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:18:00,519 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:18:02,866 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:18:03,668 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:18:06,310 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:18:06,672 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:18:08,394 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:18:12,649 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:18:13,395 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:18:14,734 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:18:17,650 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:18:18,908 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:18:23,177 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:18:23,178 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:18:24,444 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:18:24,796 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:18:28,178 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:18:29,445 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:18:33,578 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:18:33,669 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:18:34,669 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:18:34,864 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:18:38,579 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:18:39,671 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:18:42,806 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: partial_history +2023-11-23 10:18:42,807 DEBUG SenderThread:3236 [sender.py:send():380] send: history +2023-11-23 10:18:42,808 DEBUG SenderThread:3236 [sender.py:send_request():407] send_request: summary_record +2023-11-23 10:18:42,811 INFO SenderThread:3236 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-23 10:18:42,914 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/wandb-summary.json +2023-11-23 10:18:43,920 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:18:44,500 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:18:44,813 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:18:44,925 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:18:49,502 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:18:49,815 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:18:55,123 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:18:55,245 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:18:55,246 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:18:55,447 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:19:00,449 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:19:00,463 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:19:03,177 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:19:03,669 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:19:05,464 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:19:05,670 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:19:10,671 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:19:10,874 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:19:10,874 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:19:13,250 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:19:15,875 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:19:16,158 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:19:21,245 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:19:21,633 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:19:23,320 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:19:26,393 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:19:26,634 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:19:31,374 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:19:31,675 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:19:31,878 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:19:33,669 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:19:37,671 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:19:38,091 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:19:41,442 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:19:43,092 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:19:43,301 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:19:48,302 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:19:48,400 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:19:50,511 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:19:53,400 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:19:53,641 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:19:59,280 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:19:59,537 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:20:00,653 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:20:03,670 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:20:04,280 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:20:04,671 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:20:08,724 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:20:09,558 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:20:10,666 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:20:15,665 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:20:15,667 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:20:18,802 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:20:20,667 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:20:20,819 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:20:25,820 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:20:25,881 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:20:25,882 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:20:26,857 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:20:30,882 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:20:31,096 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:20:33,672 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:20:36,557 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:20:36,778 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:20:36,925 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:20:41,598 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:20:41,779 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:20:46,818 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:20:47,088 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:20:47,577 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:20:51,819 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:20:53,971 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:20:53,972 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:20:57,163 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:20:57,187 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:20:58,973 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:21:02,189 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:21:03,672 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:21:04,372 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:21:07,234 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:21:07,552 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:21:09,373 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:21:12,554 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:21:14,629 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:21:15,291 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:21:17,868 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:21:19,630 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:21:22,869 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:21:25,058 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:21:25,350 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:21:28,273 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:21:31,377 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:21:33,616 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:21:33,672 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:21:35,405 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:21:36,378 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:21:38,673 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:21:42,221 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:21:44,382 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:21:45,471 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:21:47,222 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:21:49,383 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:21:52,051 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: partial_history +2023-11-23 10:21:52,052 DEBUG SenderThread:3236 [sender.py:send():380] send: history +2023-11-23 10:21:52,053 DEBUG SenderThread:3236 [sender.py:send_request():407] send_request: summary_record +2023-11-23 10:21:52,055 INFO SenderThread:3236 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-23 10:21:52,519 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/wandb-summary.json +2023-11-23 10:21:52,648 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:21:53,523 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:21:54,532 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:21:55,057 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:21:57,649 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:22:00,058 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:22:03,177 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:22:03,589 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:22:03,673 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:22:05,674 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:22:09,511 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:22:09,512 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:22:11,643 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:22:12,659 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:22:14,512 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:22:16,644 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:22:19,772 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:22:20,708 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:22:22,049 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:22:24,773 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:22:27,050 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:22:30,093 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:22:30,769 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:22:32,258 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:22:33,673 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:22:35,094 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:22:37,675 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:22:40,841 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:22:40,927 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:22:43,074 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:22:47,284 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:22:48,075 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:22:50,896 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:22:52,285 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:22:53,538 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:22:57,794 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:22:57,794 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:22:58,957 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:22:59,024 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:23:02,795 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:23:03,674 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:23:04,675 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:23:07,927 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:23:09,023 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:23:10,350 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:23:13,167 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:23:15,352 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:23:18,672 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:23:19,092 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:23:20,823 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:23:23,674 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:23:25,824 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:23:29,155 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:23:29,353 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:23:31,527 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:23:33,677 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:23:34,716 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:23:36,873 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:23:37,209 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:23:39,718 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:23:41,874 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:23:45,121 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:23:47,361 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:23:47,371 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:23:50,121 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:23:52,373 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:23:55,764 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:23:55,764 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:23:57,422 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:23:57,952 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:24:00,765 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:24:02,953 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:24:03,676 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:24:06,049 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:24:07,487 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:24:08,720 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:24:11,050 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:24:13,721 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:24:15,546 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:24:16,138 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:24:19,320 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:24:22,428 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:24:24,680 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:24:25,609 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:24:27,433 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:24:29,681 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:24:32,492 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:24:33,677 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:24:34,756 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:24:35,680 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:24:37,494 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:24:39,756 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:24:43,140 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:24:43,140 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:24:45,349 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:24:45,739 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:24:48,140 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:24:50,350 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:24:53,764 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:24:54,800 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:24:56,009 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:24:58,820 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:25:01,010 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:25:03,098 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: partial_history +2023-11-23 10:25:03,100 DEBUG SenderThread:3236 [sender.py:send():380] send: history +2023-11-23 10:25:03,100 DEBUG SenderThread:3236 [sender.py:send_request():407] send_request: summary_record +2023-11-23 10:25:03,103 INFO SenderThread:3236 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-23 10:25:03,677 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:25:03,871 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/wandb-summary.json +2023-11-23 10:25:04,709 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:25:04,877 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:25:06,678 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:25:09,711 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:25:11,680 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:25:14,951 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:25:15,556 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:25:17,449 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:25:21,984 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:25:22,451 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:25:25,021 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:25:26,985 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:25:28,404 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:25:32,610 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:25:33,678 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:25:33,678 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:25:35,089 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:25:37,612 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:25:38,679 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:25:43,107 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:25:43,108 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:25:44,068 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:25:45,146 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:25:48,120 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:25:49,069 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:25:53,765 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:25:53,765 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:25:54,225 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:25:55,209 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:25:59,077 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:25:59,226 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:26:03,264 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:26:03,678 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:26:04,290 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:26:04,679 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:26:09,291 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:26:09,680 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:26:13,331 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:26:15,253 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:26:15,346 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:26:20,348 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:26:20,442 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:26:23,390 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:26:25,443 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:26:25,816 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:26:30,817 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:26:30,832 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:26:33,455 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:26:33,679 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:26:35,833 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:26:36,680 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:26:41,195 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:26:41,511 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:26:42,381 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:26:46,196 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:26:47,382 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:26:51,449 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:26:51,573 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:26:52,691 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:26:56,450 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:26:57,693 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:27:01,629 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:27:01,940 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:27:03,184 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:27:03,679 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:27:08,326 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:27:08,681 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:27:11,691 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:27:13,327 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:27:14,524 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:27:18,828 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:27:20,226 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:27:21,750 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:27:23,829 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:27:25,227 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:27:29,395 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:27:29,809 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:27:30,796 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:27:33,680 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:27:34,396 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:27:36,681 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:27:39,436 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:27:39,887 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:27:41,744 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:27:44,437 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:27:46,746 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:27:49,954 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:27:50,050 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:27:52,266 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:27:56,535 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:27:56,535 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:27:57,267 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:28:00,018 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:28:01,536 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:28:02,842 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:28:03,680 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:28:07,018 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:28:08,393 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:28:09,079 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:28:12,026 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:28:13,394 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:28:17,149 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: partial_history +2023-11-23 10:28:17,151 DEBUG SenderThread:3236 [sender.py:send():380] send: history +2023-11-23 10:28:17,151 DEBUG SenderThread:3236 [sender.py:send_request():407] send_request: summary_record +2023-11-23 10:28:17,154 INFO SenderThread:3236 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-23 10:28:17,968 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:28:18,143 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:28:18,146 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/wandb-summary.json +2023-11-23 10:28:19,149 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:28:19,156 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:28:22,969 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:28:24,158 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:28:27,228 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:28:28,271 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:28:29,617 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:28:33,273 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:28:33,695 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:28:34,698 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:28:38,325 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:28:39,313 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:28:39,942 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:28:43,326 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:28:44,944 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:28:47,364 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:28:48,407 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:28:50,642 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:28:54,730 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:28:55,643 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:28:57,431 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:28:59,732 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:29:00,966 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:29:03,683 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:29:05,099 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:29:06,298 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:29:07,507 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:29:10,100 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:29:11,298 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:29:15,322 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:29:15,566 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:29:16,486 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:29:20,322 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:29:21,487 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:29:25,323 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:29:26,488 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:29:30,840 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:29:31,651 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:29:32,274 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:29:33,683 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:29:35,842 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:29:37,685 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:29:40,847 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:29:41,712 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:29:43,383 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:29:45,866 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:29:48,384 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:29:51,103 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:29:53,385 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:29:56,104 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:29:58,386 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:30:01,334 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:30:01,838 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:30:03,684 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:30:03,685 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:30:06,336 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:30:08,688 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:30:11,617 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:30:11,909 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:30:14,058 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:30:16,622 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:30:19,060 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:30:21,972 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:30:22,475 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:30:24,678 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:30:27,984 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:30:30,253 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:30:32,046 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:30:32,985 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:30:33,685 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:30:35,686 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:30:38,975 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:30:41,248 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:30:42,122 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:30:43,976 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:30:46,249 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:30:49,373 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:30:50,169 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:30:51,576 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:30:54,374 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:30:56,578 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:30:59,637 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:30:59,638 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:31:00,235 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:31:01,835 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:31:03,685 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:31:06,008 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:31:07,475 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:31:09,293 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:31:11,009 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:31:12,476 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:31:16,957 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:31:18,304 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:31:19,387 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:31:21,958 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:31:23,305 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:31:27,504 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:31:27,637 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:31:28,867 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:31:32,702 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:31:33,686 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:31:34,687 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:31:37,978 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:31:39,606 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:31:40,169 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:31:42,979 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:31:45,170 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:31:47,520 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: partial_history +2023-11-23 10:31:47,522 DEBUG SenderThread:3236 [sender.py:send():380] send: history +2023-11-23 10:31:47,523 DEBUG SenderThread:3236 [sender.py:send_request():407] send_request: summary_record +2023-11-23 10:31:47,525 INFO SenderThread:3236 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-23 10:31:47,665 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/wandb-summary.json +2023-11-23 10:31:48,347 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:31:48,671 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:31:49,678 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:31:50,526 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:31:53,348 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:31:55,528 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:31:58,638 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:31:59,737 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:32:01,438 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:32:03,686 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:32:04,056 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:32:06,688 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:32:07,790 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:32:09,713 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:32:11,972 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:32:14,715 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:32:16,974 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:32:19,874 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:32:20,262 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:32:22,154 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:32:26,770 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:32:26,771 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:32:26,771 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:32:27,156 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:32:29,943 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:32:31,772 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:32:32,854 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:32:33,687 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:32:37,080 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:32:38,005 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:32:38,182 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:32:42,080 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:32:43,183 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:32:47,280 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:32:48,081 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:32:48,323 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:32:52,281 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:32:53,325 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:32:57,408 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:32:58,152 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:32:58,616 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:33:02,410 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:33:03,618 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:33:03,690 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:33:08,130 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:33:08,225 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:33:09,364 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:33:13,578 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:33:13,578 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:33:14,365 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:33:16,289 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:33:18,579 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:33:19,753 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:33:23,983 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:33:25,307 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:33:26,349 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:33:28,984 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:33:30,308 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:33:33,689 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:33:34,863 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:33:36,083 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:33:36,413 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:33:39,865 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:33:41,085 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:33:45,371 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:33:45,372 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:33:45,480 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:33:46,526 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:33:50,373 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:33:51,527 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:33:55,548 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:33:55,848 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:33:57,210 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:34:00,849 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:34:02,212 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:34:03,690 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:34:05,615 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:34:06,400 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:34:07,691 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:34:12,404 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:34:12,693 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:34:15,677 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:34:17,405 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:34:18,619 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:34:22,911 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:34:23,735 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:34:24,129 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:34:27,912 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:34:29,130 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:34:33,584 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:34:33,690 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:34:33,805 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:34:34,691 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:34:38,585 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:34:39,817 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:34:39,837 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:34:40,842 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:34:43,586 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:34:44,818 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:34:45,867 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:34:46,874 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:34:48,587 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:34:50,275 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:34:54,190 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:34:54,317 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: partial_history +2023-11-23 10:34:54,321 DEBUG SenderThread:3236 [sender.py:send():380] send: history +2023-11-23 10:34:54,322 DEBUG SenderThread:3236 [sender.py:send_request():407] send_request: summary_record +2023-11-23 10:34:54,324 INFO SenderThread:3236 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-23 10:34:54,923 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:34:54,924 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/wandb-summary.json +2023-11-23 10:34:55,327 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:34:55,927 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:34:59,191 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:35:00,327 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:35:03,691 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:35:04,478 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:35:05,709 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:35:05,993 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:35:09,478 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:35:10,710 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:35:14,052 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:35:14,807 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:35:16,006 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:35:21,008 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:35:21,131 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:35:24,121 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:35:26,132 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:35:26,313 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:35:31,314 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:35:31,368 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:35:31,369 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:35:33,691 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:35:34,187 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:35:36,408 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:35:36,692 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:35:42,283 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:35:42,535 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:35:44,253 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:35:47,284 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:35:47,536 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:35:52,306 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:35:52,575 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:35:52,756 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:35:57,757 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:35:59,056 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:35:59,057 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:36:02,370 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:36:03,354 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:36:03,692 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:36:04,056 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:36:08,693 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:36:09,682 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:36:12,441 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:36:13,927 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:36:14,683 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:36:18,928 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:36:20,085 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:36:20,497 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:36:24,266 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:36:25,086 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:36:30,038 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:36:30,383 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:36:30,571 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:36:33,692 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:36:35,384 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:36:35,693 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:36:39,637 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:36:41,025 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:36:41,203 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:36:46,205 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:36:47,376 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:36:47,377 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:36:49,709 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:36:51,522 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:36:52,378 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:36:56,523 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:36:57,406 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:36:57,407 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:36:59,781 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:37:01,965 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:37:02,595 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:37:03,693 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:37:07,413 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:37:07,834 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:37:08,081 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:37:12,414 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:37:13,082 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:37:17,796 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:37:17,896 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:37:18,602 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:37:22,798 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:37:23,604 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:37:27,963 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:37:28,711 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:37:29,407 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:37:33,694 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:37:34,696 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:37:34,932 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:37:38,042 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:37:39,933 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:37:40,165 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:37:45,166 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:37:45,824 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:37:45,824 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:37:48,115 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:37:50,183 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:37:50,825 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:37:55,184 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:37:56,173 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:37:56,312 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:38:00,673 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:38:01,313 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:38:03,708 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:38:04,921 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: partial_history +2023-11-23 10:38:04,922 DEBUG SenderThread:3236 [sender.py:send():380] send: history +2023-11-23 10:38:04,923 DEBUG SenderThread:3236 [sender.py:send_request():407] send_request: summary_record +2023-11-23 10:38:04,926 INFO SenderThread:3236 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-23 10:38:05,230 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/wandb-summary.json +2023-11-23 10:38:05,927 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:38:06,236 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:38:06,741 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:38:07,242 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:38:10,928 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:38:11,743 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:38:16,311 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:38:16,885 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:38:17,578 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:38:21,887 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:38:23,149 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:38:26,387 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:38:27,398 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:38:28,150 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:38:32,399 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:38:33,696 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:38:33,760 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:38:36,454 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:38:37,915 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:38:38,761 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:38:42,917 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:38:44,230 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:38:44,511 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:38:48,456 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:38:49,231 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:38:54,395 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:38:54,578 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:38:54,611 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:38:59,395 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:38:59,612 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:39:03,696 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:39:04,697 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:39:04,745 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:39:05,264 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:39:09,698 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:39:10,266 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:39:14,807 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:39:15,328 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:39:15,579 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:39:20,330 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:39:21,507 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:39:23,870 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:39:25,721 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:39:26,509 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:39:30,722 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:39:32,164 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:39:32,165 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:39:33,697 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:39:33,930 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:39:36,698 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:39:37,165 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:39:41,699 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:39:42,760 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:39:43,993 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:39:46,959 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:39:47,761 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:39:52,042 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:39:52,703 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:39:53,515 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:39:56,057 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:39:58,516 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:39:58,556 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:40:03,698 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:40:03,698 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:40:04,072 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:40:04,120 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:40:08,699 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:40:09,074 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:40:14,187 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:40:14,240 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:40:14,579 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:40:19,241 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:40:20,605 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:40:22,242 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:40:24,739 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:40:25,606 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:40:29,740 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:40:31,241 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:40:31,242 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:40:32,315 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:40:33,698 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:40:35,699 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:40:36,242 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:40:40,700 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:40:41,841 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:40:42,382 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:40:46,059 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:40:46,842 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:40:51,865 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:40:52,058 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:40:52,442 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:40:56,866 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:40:57,059 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:41:02,479 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:41:02,480 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:41:02,522 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:41:02,722 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:41:03,699 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:41:08,700 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:41:08,950 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:41:12,582 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:41:13,951 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:41:14,264 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:41:19,265 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:41:19,468 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:41:19,658 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: partial_history +2023-11-23 10:41:19,659 DEBUG SenderThread:3236 [sender.py:send():380] send: history +2023-11-23 10:41:19,660 DEBUG SenderThread:3236 [sender.py:send_request():407] send_request: summary_record +2023-11-23 10:41:19,663 INFO SenderThread:3236 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-23 10:41:20,653 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/wandb-summary.json +2023-11-23 10:41:21,660 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:41:22,664 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:41:24,469 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:41:24,665 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:41:29,479 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:41:29,479 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:41:30,047 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:41:30,723 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:41:33,699 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:41:34,835 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:41:35,700 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:41:40,136 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:41:40,784 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:41:41,339 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:41:45,137 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:41:46,341 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:41:50,299 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:41:50,856 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:41:51,646 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:41:55,301 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:41:56,647 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:42:00,479 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:42:00,917 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:42:02,390 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:42:03,700 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:42:06,688 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:42:07,701 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:42:10,991 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:42:11,690 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:42:12,952 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:42:17,351 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:42:17,352 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:42:18,549 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:42:20,051 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:42:22,352 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:42:23,550 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:42:27,480 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:42:28,648 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:42:30,117 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:42:32,481 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:42:32,482 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:42:33,649 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:42:33,700 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:42:38,314 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:42:39,561 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:42:40,181 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:42:43,315 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:42:44,562 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:42:48,242 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:42:48,863 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:42:48,863 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:42:50,256 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:42:55,258 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:42:55,363 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:42:58,306 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:43:00,364 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:43:00,583 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:43:03,703 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:43:05,705 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:43:05,837 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:43:05,838 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:43:08,373 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:43:10,839 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:43:11,167 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:43:15,863 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:43:16,757 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:43:18,439 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:43:21,496 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:43:21,758 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:43:26,740 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:43:26,957 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:43:28,503 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:43:31,741 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:43:31,958 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:43:33,704 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:43:36,578 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:43:37,344 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:43:37,638 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:43:42,345 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:43:42,639 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:43:47,347 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:43:47,641 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:43:52,696 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:43:53,103 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:43:53,565 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:43:58,567 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:43:59,197 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:44:02,764 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:44:03,704 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:44:03,705 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:44:04,198 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:44:08,707 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:44:09,474 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:44:10,825 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:44:14,475 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:44:14,673 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:44:19,960 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:44:20,603 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:44:20,895 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:44:24,961 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:44:25,604 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:44:30,217 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:44:30,958 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:44:31,450 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:44:33,705 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:44:35,218 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:44:36,706 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:44:38,885 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: partial_history +2023-11-23 10:44:38,886 DEBUG SenderThread:3236 [sender.py:send():380] send: history +2023-11-23 10:44:38,886 DEBUG SenderThread:3236 [sender.py:send_request():407] send_request: summary_record +2023-11-23 10:44:38,889 INFO SenderThread:3236 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-23 10:44:39,009 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/wandb-summary.json +2023-11-23 10:44:40,016 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:44:40,608 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:44:41,024 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:44:41,891 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:44:45,610 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:44:46,892 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:44:51,093 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:44:51,572 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:44:52,861 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:44:56,962 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:44:57,862 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:45:01,172 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:45:01,963 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:45:03,215 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:45:03,706 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:45:07,471 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:45:08,740 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:45:09,221 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:45:12,472 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:45:13,741 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:45:17,776 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:45:18,280 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:45:18,966 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:45:22,777 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:45:23,968 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:45:27,881 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:45:28,350 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:45:28,971 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:45:32,883 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:45:33,707 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:45:34,709 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:45:38,432 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:45:38,880 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:45:40,053 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:45:44,525 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:45:45,055 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:45:48,496 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:45:49,526 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:45:50,847 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:45:55,099 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:45:56,289 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:45:56,553 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:46:00,100 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:46:01,289 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:46:03,709 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:46:05,718 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:46:05,718 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:46:06,616 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:46:06,873 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:46:10,719 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:46:11,874 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:46:15,865 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:46:15,866 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:46:16,691 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:46:16,936 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:46:21,537 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:46:21,937 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:46:26,752 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:46:27,166 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:46:27,359 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:46:32,361 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:46:32,448 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:46:33,709 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:46:34,810 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:46:37,450 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:46:37,710 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:46:42,711 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:46:42,763 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:46:44,882 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:46:47,764 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:46:47,966 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:46:53,250 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:46:53,393 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:46:54,939 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:46:58,251 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:46:58,394 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:47:03,699 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:47:03,709 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:47:03,710 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:47:05,005 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:47:08,701 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:47:08,723 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:47:13,058 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:47:13,795 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:47:14,463 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:47:18,797 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:47:20,127 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:47:23,116 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:47:24,631 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:47:25,128 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:47:29,632 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:47:30,866 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:47:30,867 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:47:33,185 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:47:33,720 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:47:34,721 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:47:35,950 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:47:39,722 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:47:41,319 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:47:43,279 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:47:45,516 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:47:46,320 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:47:50,141 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: partial_history +2023-11-23 10:47:50,145 DEBUG SenderThread:3236 [sender.py:send():380] send: history +2023-11-23 10:47:50,145 DEBUG SenderThread:3236 [sender.py:send_request():407] send_request: summary_record +2023-11-23 10:47:50,148 INFO SenderThread:3236 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-23 10:47:50,331 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/wandb-summary.json +2023-11-23 10:47:51,149 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:47:51,985 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:47:52,342 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:47:53,347 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:47:56,150 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:47:58,291 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:48:01,401 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:48:01,512 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:48:03,292 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:48:03,710 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:48:06,712 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:48:08,690 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:48:10,464 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:48:11,933 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:48:13,690 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:48:16,934 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:48:19,076 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:48:19,076 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:48:20,539 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:48:22,315 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:48:24,077 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:48:27,317 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:48:28,593 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:48:29,527 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:48:32,757 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:48:33,711 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:48:34,529 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:48:38,632 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:48:38,667 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:48:40,400 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:48:43,633 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:48:45,720 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:48:48,731 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:48:49,075 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:48:50,722 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:48:54,076 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:48:56,253 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:48:58,796 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:48:59,437 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:49:01,254 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:49:03,712 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:49:04,714 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:49:06,557 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:49:06,557 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:49:06,889 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:49:09,777 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:49:11,558 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:49:14,778 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:49:16,920 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:49:16,954 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:49:20,163 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:49:21,922 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:49:25,781 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:49:27,027 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:49:27,660 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:49:30,782 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:49:32,938 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:49:33,712 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:49:36,218 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:49:37,095 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:49:37,939 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:49:41,219 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:49:43,229 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:49:45,159 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:49:46,349 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:49:48,230 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:49:51,351 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:49:53,450 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:49:55,224 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:49:56,612 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:49:58,451 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:50:01,613 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:50:03,713 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:50:03,925 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:50:05,291 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:50:06,714 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:50:10,310 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:50:11,715 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:50:13,340 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:50:15,311 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:50:16,745 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:50:20,532 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:50:20,532 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:50:21,747 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:50:23,399 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:50:25,533 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:50:26,992 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:50:31,247 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:50:32,457 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:50:33,467 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:50:33,713 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:50:36,248 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:50:37,714 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:50:41,892 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:50:42,922 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:50:43,537 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:50:46,895 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:50:47,924 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:50:52,427 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:50:53,606 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:50:53,612 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:50:58,607 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:50:58,850 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:51:00,062 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: partial_history +2023-11-23 10:51:00,064 DEBUG SenderThread:3236 [sender.py:send():380] send: history +2023-11-23 10:51:00,064 DEBUG SenderThread:3236 [sender.py:send_request():407] send_request: summary_record +2023-11-23 10:51:00,067 INFO SenderThread:3236 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-23 10:51:00,657 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:51:00,658 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/wandb-summary.json +2023-11-23 10:51:01,662 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:51:03,714 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:51:03,717 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:51:03,851 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:51:08,718 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:51:09,692 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:51:10,713 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:51:14,091 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:51:14,730 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:51:19,092 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:51:20,427 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:51:20,785 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:51:24,933 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:51:25,581 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:51:30,736 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:51:30,848 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:51:31,289 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:51:33,715 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:51:36,356 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:51:36,716 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:51:40,911 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:51:42,341 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:51:42,589 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:51:47,591 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:51:48,118 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:51:50,975 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:51:53,120 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:51:53,343 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:51:58,345 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:51:59,084 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:52:01,039 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:52:03,715 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:52:03,716 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:52:04,085 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:52:08,717 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:52:09,512 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:52:11,098 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:52:13,762 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:52:14,513 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:52:19,399 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:52:20,092 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:52:21,170 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:52:24,400 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:52:25,095 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:52:29,226 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:52:29,867 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:52:30,771 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:52:33,716 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:52:35,717 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:52:36,027 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:52:36,027 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:52:39,286 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:52:41,028 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:52:41,292 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:52:46,294 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:52:46,790 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:52:49,347 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:52:51,791 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:52:51,981 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:52:57,379 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:52:57,479 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:52:59,424 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:53:02,379 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:53:02,480 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:53:03,718 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:53:07,517 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:53:07,517 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:53:07,753 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:53:09,489 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:53:12,518 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:53:12,754 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:53:18,213 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:53:18,633 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:53:19,558 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:53:23,215 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:53:23,634 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:53:27,607 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:53:29,082 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:53:29,222 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:53:33,720 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:53:34,454 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:53:34,722 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:53:37,674 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:53:39,455 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:53:40,660 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:53:44,867 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:53:46,096 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:53:47,763 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:53:49,868 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:53:51,097 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:53:55,647 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:53:56,826 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:53:57,829 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:54:00,648 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:54:01,827 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:54:03,720 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:54:05,706 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:54:06,916 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:54:07,827 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:54:10,707 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:54:12,829 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:54:14,306 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: partial_history +2023-11-23 10:54:14,307 DEBUG SenderThread:3236 [sender.py:send():380] send: history +2023-11-23 10:54:14,308 DEBUG SenderThread:3236 [sender.py:send_request():407] send_request: summary_record +2023-11-23 10:54:14,310 INFO SenderThread:3236 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-23 10:54:14,968 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:54:14,968 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/wandb-summary.json +2023-11-23 10:54:15,974 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:54:16,007 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:54:18,313 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:54:21,008 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:54:23,314 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:54:25,038 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:54:26,876 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:54:29,086 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:54:32,430 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:54:33,722 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:54:34,724 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:54:35,099 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:54:37,435 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:54:39,725 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:54:42,575 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:54:44,742 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:54:45,167 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:54:47,576 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:54:49,743 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:54:52,885 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:54:52,885 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:54:55,140 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:54:55,238 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:54:57,889 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:55:00,141 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:55:03,463 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:55:03,721 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:55:05,302 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:55:05,722 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:55:08,464 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:55:10,724 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:55:13,355 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:55:13,823 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:55:16,001 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:55:20,113 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:55:21,002 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:55:23,426 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:55:25,114 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:55:26,285 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:55:30,742 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:55:31,878 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:55:33,487 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:55:33,722 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:55:35,743 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:55:37,724 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:55:41,344 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:55:41,344 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:55:41,554 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:55:43,530 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:55:46,345 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:55:48,532 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:55:51,626 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:55:51,885 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:55:51,885 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:55:54,188 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:55:57,035 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:55:59,190 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:56:01,686 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:56:02,634 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:56:03,722 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:56:04,723 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:56:08,263 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:56:08,264 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:56:08,266 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:56:10,502 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:56:11,758 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:56:13,265 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:56:15,503 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:56:18,788 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:56:21,076 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:56:21,936 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:56:23,789 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:56:26,078 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:56:29,633 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:56:31,750 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:56:32,003 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:56:33,723 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:56:34,634 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:56:37,725 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:56:39,057 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:56:40,063 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:56:43,406 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:56:45,065 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:56:48,946 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:56:49,123 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:56:50,670 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:56:53,947 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:56:56,465 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:56:56,467 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:56:59,197 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:56:59,747 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:57:01,466 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:57:03,734 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:57:05,736 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:57:07,111 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:57:09,268 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:57:11,334 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:57:12,113 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:57:16,335 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:57:17,519 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:57:19,323 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:57:21,767 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:57:22,538 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:57:26,769 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:57:26,811 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: partial_history +2023-11-23 10:57:26,813 DEBUG SenderThread:3236 [sender.py:send():380] send: history +2023-11-23 10:57:26,814 DEBUG SenderThread:3236 [sender.py:send_request():407] send_request: summary_record +2023-11-23 10:57:26,816 INFO SenderThread:3236 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-23 10:57:27,376 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/wandb-summary.json +2023-11-23 10:57:27,618 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:57:27,619 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:57:28,381 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:57:29,390 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:57:31,819 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:57:32,619 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:57:33,725 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:57:37,302 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:57:37,452 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:57:38,044 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:57:42,305 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:57:43,046 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:57:47,516 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:57:48,139 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:57:48,919 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:57:53,139 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:57:53,920 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:57:58,140 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:57:59,454 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:58:03,614 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:58:03,725 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:58:03,726 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:58:04,455 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:58:08,727 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:58:09,753 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:58:09,753 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:58:13,680 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:58:14,218 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:58:14,754 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:58:19,220 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:58:19,914 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:58:23,745 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:58:24,250 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:58:24,915 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:58:29,251 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:58:30,486 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:58:33,726 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:58:33,826 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:58:34,727 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:58:35,487 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:58:39,728 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:58:40,708 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:58:43,901 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:58:44,907 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:58:45,708 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:58:49,909 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:58:51,232 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:58:52,036 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:58:55,610 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:58:56,232 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:59:01,317 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:59:01,784 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:59:02,111 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:59:03,726 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:59:06,727 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:59:06,785 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:59:12,182 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:59:12,258 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:59:12,566 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:59:17,259 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:59:17,567 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:59:22,869 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:59:23,241 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:59:23,417 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:59:27,871 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:59:28,242 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:59:33,477 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:59:33,727 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 10:59:33,728 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:59:33,743 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:59:38,728 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:59:39,400 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:59:41,540 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:59:44,401 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:59:44,524 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:59:49,525 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 10:59:49,842 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:59:51,609 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 10:59:54,842 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 10:59:55,098 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:00:00,630 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:00:00,859 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:00:01,681 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:00:03,727 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:00:05,631 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:00:06,729 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:00:10,755 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:00:11,748 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:00:12,164 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:00:15,961 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:00:17,166 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:00:19,814 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:00:21,634 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:00:22,858 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:00:27,088 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:00:27,860 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:00:29,881 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:00:32,089 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:00:33,357 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:00:33,728 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:00:37,685 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:00:38,884 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:00:39,954 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:00:42,686 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:00:43,885 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:00:47,924 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: partial_history +2023-11-23 11:00:47,926 DEBUG SenderThread:3236 [sender.py:send():380] send: history +2023-11-23 11:00:47,926 DEBUG SenderThread:3236 [sender.py:send_request():407] send_request: summary_record +2023-11-23 11:00:47,929 INFO SenderThread:3236 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-23 11:00:48,015 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/wandb-summary.json +2023-11-23 11:00:48,643 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:00:48,930 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:00:50,028 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:00:51,033 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:00:53,644 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:00:53,931 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:00:58,814 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:00:58,815 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:00:58,981 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:01:00,094 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:01:03,728 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:01:03,815 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:01:04,729 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:01:09,185 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:01:10,159 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:01:10,380 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:01:15,381 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:01:15,603 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:01:18,216 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:01:20,605 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:01:20,910 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:01:25,911 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:01:26,266 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:01:26,267 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:01:28,281 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:01:31,267 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:01:31,418 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:01:33,730 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:01:36,539 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:01:36,678 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:01:38,345 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:01:41,540 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:01:41,679 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:01:47,313 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:01:47,644 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:01:48,417 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:01:52,315 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:01:52,660 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:01:57,484 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:01:57,956 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:01:58,219 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:02:03,220 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:02:03,386 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:02:03,731 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:02:05,542 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:02:08,387 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:02:08,560 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:02:13,561 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:02:13,724 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:02:13,724 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:02:15,620 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:02:18,725 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:02:18,997 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:02:23,766 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:02:24,988 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:02:25,683 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:02:28,767 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:02:29,989 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:02:33,730 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:02:34,222 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:02:35,738 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:02:35,752 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:02:39,223 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:02:40,740 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:02:44,933 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:02:44,933 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:02:45,816 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:02:46,178 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:02:51,179 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:02:51,488 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:02:53,871 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:02:56,489 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:02:56,764 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:03:01,766 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:03:01,917 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:03:03,731 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:03:03,943 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:03:06,918 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:03:07,732 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:03:12,552 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:03:13,696 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:03:14,004 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:03:17,553 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:03:18,698 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:03:23,277 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:03:24,087 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:03:24,498 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:03:29,500 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:03:29,773 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:03:29,775 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:03:32,158 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:03:33,731 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:03:34,732 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:03:34,775 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:03:39,733 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:03:40,105 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:03:42,232 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:03:45,106 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:03:45,247 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:03:50,466 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:03:50,743 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:03:52,298 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:03:55,467 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:03:55,744 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:03:59,127 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: partial_history +2023-11-23 11:03:59,129 DEBUG SenderThread:3236 [sender.py:send():380] send: history +2023-11-23 11:03:59,129 DEBUG SenderThread:3236 [sender.py:send_request():407] send_request: summary_record +2023-11-23 11:03:59,132 INFO SenderThread:3236 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-23 11:03:59,347 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/wandb-summary.json +2023-11-23 11:04:00,352 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:04:00,868 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:04:01,134 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:04:01,357 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:04:03,732 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:04:06,733 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:04:07,218 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:04:10,423 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:04:12,219 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:04:12,338 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:04:17,339 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:04:17,762 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:04:20,484 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:04:22,762 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:04:22,953 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:04:28,111 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:04:28,321 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:04:30,557 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:04:33,112 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:04:33,322 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:04:33,732 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:04:38,610 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:04:38,928 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:04:39,093 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:04:44,095 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:04:45,259 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:04:45,259 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:04:47,685 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:04:49,552 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:04:50,260 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:04:54,553 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:04:55,850 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:04:57,745 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:05:00,082 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:05:00,851 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:05:03,735 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:05:05,395 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:05:06,199 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:05:07,812 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:05:10,396 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:05:11,200 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:05:15,398 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:05:16,568 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:05:17,879 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:05:20,741 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:05:21,570 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:05:26,402 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:05:27,036 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:05:27,941 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:05:31,403 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:05:32,038 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:05:33,737 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:05:36,009 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:05:37,054 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:05:37,664 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:05:42,056 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:05:43,362 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:05:43,362 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:05:46,073 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:05:47,600 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:05:48,363 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:05:52,602 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:05:53,902 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:05:56,133 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:05:58,164 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:05:58,902 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:06:03,165 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:06:03,736 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:06:04,596 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:06:06,197 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:06:08,738 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:06:09,597 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:06:14,423 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:06:15,252 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:06:16,261 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:06:19,424 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:06:21,769 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:06:24,330 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:06:25,084 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:06:26,770 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:06:30,085 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:06:32,518 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:06:33,763 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:06:34,395 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:06:35,764 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:06:37,518 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:06:40,766 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:06:43,149 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:06:44,477 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:06:46,611 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:06:48,200 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:06:51,613 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:06:53,944 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:06:54,539 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:06:57,145 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:07:00,443 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:07:00,444 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:07:02,654 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:07:03,738 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:07:04,615 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:07:05,445 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:07:07,739 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:07:10,859 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:07:11,039 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: partial_history +2023-11-23 11:07:11,040 DEBUG SenderThread:3236 [sender.py:send():380] send: history +2023-11-23 11:07:11,040 DEBUG SenderThread:3236 [sender.py:send_request():407] send_request: summary_record +2023-11-23 11:07:11,043 INFO SenderThread:3236 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-23 11:07:11,656 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/wandb-summary.json +2023-11-23 11:07:12,661 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:07:13,045 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:07:13,667 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:07:15,860 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:07:18,047 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:07:21,595 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:07:22,735 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:07:23,848 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:07:26,596 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:07:28,850 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:07:32,069 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:07:32,798 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:07:33,738 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:07:34,739 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:07:38,492 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:07:40,637 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:07:41,854 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:07:43,493 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:07:45,638 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:07:48,825 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:07:49,922 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:07:51,069 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:07:53,826 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:07:56,070 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:07:59,224 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:07:59,987 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:08:01,458 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:08:03,739 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:08:04,225 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:08:06,740 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:08:10,056 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:08:10,140 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:08:12,413 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:08:16,632 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:08:16,633 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:08:17,414 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:08:20,119 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:08:21,634 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:08:22,947 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:08:27,253 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:08:28,177 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:08:30,181 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:08:32,535 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:08:33,739 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:08:33,740 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:08:38,339 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:08:38,742 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:08:42,261 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:08:43,340 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:08:43,749 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:08:48,750 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:08:49,192 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:08:50,322 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:08:54,249 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:08:54,362 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:08:59,347 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:08:59,348 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:08:59,363 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:09:00,396 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:09:03,748 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:09:04,348 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:09:04,749 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:09:09,785 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:09:10,095 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:09:10,462 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:09:14,786 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:09:15,096 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:09:20,192 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:09:20,366 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:09:20,562 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:09:25,202 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:09:25,368 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:09:30,357 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:09:30,621 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:09:31,020 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:09:33,749 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:09:36,487 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:09:36,750 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:09:40,693 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:09:41,488 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:09:41,779 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:09:46,781 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:09:47,375 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:09:47,376 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:09:50,762 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:09:52,088 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:09:52,376 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:09:57,089 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:09:57,610 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:10:00,837 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:10:02,611 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:10:02,795 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:10:03,750 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:10:07,993 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:10:08,233 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:10:08,892 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:10:12,993 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:10:13,234 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:10:18,586 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:10:18,727 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:10:18,960 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:10:23,729 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:10:25,145 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:10:26,334 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: partial_history +2023-11-23 11:10:26,335 DEBUG SenderThread:3236 [sender.py:send():380] send: history +2023-11-23 11:10:26,335 DEBUG SenderThread:3236 [sender.py:send_request():407] send_request: summary_record +2023-11-23 11:10:26,338 INFO SenderThread:3236 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-23 11:10:27,021 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:10:27,021 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/wandb-summary.json +2023-11-23 11:10:29,032 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:10:29,340 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:10:30,146 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:10:33,751 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:10:34,753 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:10:35,916 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:10:38,099 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:10:40,120 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:10:40,917 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:10:45,122 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:10:45,998 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:10:45,999 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:10:48,168 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:10:50,397 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:10:50,999 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:10:55,399 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:10:56,555 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:10:58,225 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:11:00,811 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:11:01,556 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:11:03,752 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:11:06,161 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:11:06,280 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:11:06,940 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:11:11,162 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:11:13,403 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:11:16,339 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:11:16,603 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:11:18,405 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:11:21,604 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:11:23,709 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:11:26,404 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:11:26,929 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:11:28,710 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:11:31,930 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:11:33,752 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:11:34,353 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:11:34,470 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:11:37,754 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:11:39,354 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:11:42,755 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:11:44,404 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:11:44,538 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:11:48,257 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:11:50,127 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:11:53,258 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:11:54,607 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:11:55,937 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:11:58,312 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:12:01,518 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:12:01,518 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:12:03,753 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:12:03,753 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:12:04,674 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:12:06,519 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:12:08,754 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:12:12,476 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:12:14,692 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:12:14,741 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:12:17,477 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:12:19,692 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:12:22,478 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:12:24,694 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:12:27,542 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:12:28,815 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:12:29,988 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:12:32,543 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:12:33,757 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:12:35,760 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:12:37,573 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:12:38,881 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:12:41,505 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:12:42,574 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:12:46,507 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:12:47,957 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:12:47,958 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:12:48,948 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:12:51,617 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:12:52,959 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:12:56,619 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:12:58,337 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:12:59,018 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:13:01,634 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:13:03,338 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:13:03,757 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:13:06,760 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:13:08,676 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:13:09,089 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:13:11,851 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:13:14,988 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:13:14,988 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:13:17,209 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:13:19,160 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:13:19,989 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:13:22,210 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:13:25,533 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:13:27,211 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:13:27,815 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:13:30,534 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:13:32,817 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:13:33,758 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:13:36,194 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:13:37,279 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:13:38,343 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:13:41,195 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:13:43,345 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:13:45,277 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: partial_history +2023-11-23 11:13:45,279 DEBUG SenderThread:3236 [sender.py:send():380] send: history +2023-11-23 11:13:45,279 DEBUG SenderThread:3236 [sender.py:send_request():407] send_request: summary_record +2023-11-23 11:13:45,282 INFO SenderThread:3236 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-23 11:13:45,332 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/wandb-summary.json +2023-11-23 11:13:46,336 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:13:47,045 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:13:47,046 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:13:47,340 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:13:49,284 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:13:53,542 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:13:54,285 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:13:56,402 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:13:58,544 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:13:59,834 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:14:03,758 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:14:03,973 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:14:03,973 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:14:05,194 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:14:06,463 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:14:08,974 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:14:10,195 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:14:14,045 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:14:14,045 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:14:14,522 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:14:15,502 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:14:19,358 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:14:20,503 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:14:24,586 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:14:24,789 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:14:26,091 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:14:31,092 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:14:31,232 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:14:31,233 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:14:33,759 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:14:34,650 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:14:36,263 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:14:36,760 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:14:41,761 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:14:42,051 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:14:44,720 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:14:47,052 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:14:47,175 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:14:52,595 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:14:52,779 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:14:52,782 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:14:57,597 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:14:57,780 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:15:02,845 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:15:03,185 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:15:03,186 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:15:03,339 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:15:03,759 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:15:08,761 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:15:09,555 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:15:12,917 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:15:13,845 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:15:14,556 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:15:18,847 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:15:20,192 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:15:23,044 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:15:24,424 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:15:25,193 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:15:29,426 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:15:30,208 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:15:33,116 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:15:33,760 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:15:34,761 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:15:35,209 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:15:40,518 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:15:40,673 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:15:41,173 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:15:45,519 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:15:45,674 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:15:50,774 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:15:51,028 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:15:51,245 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:15:55,775 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:15:56,029 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:16:01,185 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:16:01,309 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:16:01,626 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:16:03,771 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:16:06,774 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:16:07,244 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:16:11,380 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:16:12,245 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:16:12,499 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:16:17,500 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:16:17,836 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:16:17,837 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:16:19,434 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:16:22,837 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:16:23,100 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:16:28,697 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:16:28,987 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:16:29,497 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:16:33,698 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:16:33,761 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:16:34,762 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:16:38,557 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:16:39,068 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:16:40,272 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:16:45,273 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:16:45,409 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:16:45,409 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:16:45,410 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:16:48,666 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:16:50,410 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:16:50,630 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:16:55,631 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:16:55,712 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:16:56,379 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: partial_history +2023-11-23 11:16:56,381 DEBUG SenderThread:3236 [sender.py:send():380] send: history +2023-11-23 11:16:56,381 DEBUG SenderThread:3236 [sender.py:send_request():407] send_request: summary_record +2023-11-23 11:16:56,384 INFO SenderThread:3236 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-23 11:16:56,716 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/wandb-summary.json +2023-11-23 11:16:57,722 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:16:58,727 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:17:00,713 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:17:01,386 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:17:03,762 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:17:06,598 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:17:06,852 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:17:06,876 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:17:11,599 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:17:11,852 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:17:16,665 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:17:16,878 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:17:16,954 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:17:21,666 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:17:21,879 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:17:27,023 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:17:27,237 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:17:27,494 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:17:32,495 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:17:33,658 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:17:33,659 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:17:33,764 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:17:37,103 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:17:37,916 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:17:38,660 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:17:42,918 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:17:43,989 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:17:45,503 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:17:48,110 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:17:48,990 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:17:53,112 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:17:54,356 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:17:55,575 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:17:58,612 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:17:59,357 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:18:03,764 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:18:03,765 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:18:04,636 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:18:04,655 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:18:08,766 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:18:09,657 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:18:14,514 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:18:14,701 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:18:15,332 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:18:19,528 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:18:20,899 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:18:20,899 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:18:22,758 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:18:25,090 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:18:25,900 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:18:30,091 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:18:31,447 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:18:32,823 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:18:33,765 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:18:35,766 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:18:36,448 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:18:40,767 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:18:42,111 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:18:42,884 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:18:46,275 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:18:47,112 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:18:51,703 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:18:52,484 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:18:52,945 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:18:56,704 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:18:57,500 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:19:01,919 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:19:02,655 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:19:03,001 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:19:03,765 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:19:07,767 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:19:09,370 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:19:13,060 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:19:13,562 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:19:14,371 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:19:18,563 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:19:19,373 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:19:23,563 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:19:24,374 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:19:28,564 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:19:29,706 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:19:33,565 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:19:33,766 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:19:34,707 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:19:38,767 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:19:39,708 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:19:43,768 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:19:45,105 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:19:45,696 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:19:49,165 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:19:50,106 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:19:54,166 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:19:54,765 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:19:55,954 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:19:59,261 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:20:00,956 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:20:03,766 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:20:04,833 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:20:04,955 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:20:06,698 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:20:06,699 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:20:09,956 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:20:12,819 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:20:14,903 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:20:15,010 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:20:17,820 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:20:20,011 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:20:23,535 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:20:25,775 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:20:26,986 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:20:28,537 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:20:30,776 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:20:33,768 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:20:34,355 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:20:34,355 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:20:34,639 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: partial_history +2023-11-23 11:20:34,640 DEBUG SenderThread:3236 [sender.py:send():380] send: history +2023-11-23 11:20:34,641 DEBUG SenderThread:3236 [sender.py:send_request():407] send_request: summary_record +2023-11-23 11:20:34,650 INFO SenderThread:3236 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-23 11:20:35,036 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/wandb-summary.json +2023-11-23 11:20:36,040 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:20:36,652 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:20:37,045 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:20:39,356 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:20:41,654 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:20:45,188 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:20:47,108 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:20:47,633 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:20:50,190 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:20:52,634 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:20:55,166 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:20:55,891 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:20:58,161 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:21:00,893 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:21:03,162 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:21:03,768 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:21:05,233 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:21:06,359 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:21:08,973 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:21:12,154 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:21:14,595 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:21:15,298 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:21:17,155 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:21:19,596 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:21:22,832 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:21:22,833 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:21:25,084 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:21:25,354 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:21:27,834 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:21:30,085 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:21:33,355 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:21:33,355 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:21:33,769 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:21:35,419 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:21:35,771 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:21:38,732 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:21:40,773 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:21:44,445 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:21:45,492 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:21:45,861 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:21:49,446 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:21:50,862 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:21:54,880 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:21:55,552 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:21:56,107 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:22:01,108 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:22:01,314 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:22:03,769 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:22:05,626 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:22:06,315 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:22:06,770 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:22:11,772 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:22:11,924 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:22:15,690 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:22:16,925 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:22:17,274 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:22:22,275 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:22:22,673 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:22:22,673 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:22:25,756 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:22:27,674 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:22:27,969 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:22:33,345 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:22:33,525 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:22:33,770 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:22:33,812 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:22:38,346 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:22:38,771 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:22:43,876 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:22:44,068 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:22:44,283 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:22:49,070 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:22:49,284 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:22:52,943 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:22:54,352 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:22:54,833 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:22:59,354 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:23:00,848 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:23:03,019 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:23:03,771 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:23:04,772 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:23:05,849 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:23:09,773 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:23:11,546 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:23:13,079 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:23:14,931 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:23:16,550 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:23:19,933 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:23:22,464 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:23:22,464 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:23:23,140 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:23:25,165 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:23:27,466 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:23:30,167 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:23:32,739 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:23:33,208 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:23:33,771 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:23:35,772 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:23:37,740 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:23:40,773 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:23:43,280 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:23:43,300 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:23:46,601 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:23:48,301 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:23:51,568 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: partial_history +2023-11-23 11:23:51,569 DEBUG SenderThread:3236 [sender.py:send():380] send: history +2023-11-23 11:23:51,570 DEBUG SenderThread:3236 [sender.py:send_request():407] send_request: summary_record +2023-11-23 11:23:51,573 INFO SenderThread:3236 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-23 11:23:52,338 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:23:52,339 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/wandb-summary.json +2023-11-23 11:23:52,574 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:23:53,340 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:23:53,340 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:23:53,343 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:23:57,575 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:23:59,781 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:24:03,052 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:24:03,410 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:24:03,771 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:24:04,782 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:24:08,773 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:24:09,783 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:24:13,774 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:24:14,784 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:24:18,774 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:24:20,131 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:24:20,132 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:24:21,526 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:24:24,590 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:24:25,133 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:24:29,604 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:24:30,141 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:24:31,585 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:24:33,772 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:24:34,773 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:24:35,141 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:24:39,774 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:24:40,917 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:24:41,653 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:24:45,086 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:24:45,918 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:24:50,761 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:24:51,132 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:24:51,724 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:24:55,762 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:24:58,097 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:25:01,488 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:25:01,784 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:25:03,098 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:25:03,772 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:25:06,774 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:25:08,644 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:25:08,644 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:25:09,841 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:25:11,892 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:25:13,645 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:25:16,894 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:25:19,131 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:25:19,132 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:25:19,907 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:25:22,583 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:25:24,223 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:25:27,584 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:25:29,940 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:25:29,974 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:25:33,198 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:25:33,773 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:25:34,942 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:25:39,099 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:25:40,045 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:25:40,851 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:25:44,100 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:25:46,456 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:25:49,826 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:25:50,109 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:25:51,457 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:25:54,827 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:25:56,601 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:25:59,164 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:25:59,847 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:26:01,615 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:26:03,773 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:26:05,774 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:26:06,978 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:26:06,979 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:26:09,234 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:26:11,385 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:26:11,979 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:26:16,387 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:26:17,286 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:26:17,581 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:26:21,842 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:26:22,582 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:26:27,346 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:26:27,510 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:26:27,914 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:26:32,511 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:26:32,915 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:26:33,775 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:26:37,420 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:26:37,652 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:26:38,367 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:26:42,653 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:26:43,369 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:26:47,486 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:26:48,362 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:26:49,064 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:26:53,363 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:26:54,936 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:26:54,937 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:26:57,545 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:26:59,115 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:26:59,938 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:27:03,777 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:27:04,778 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:27:05,332 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:27:07,622 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:27:10,333 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:27:10,701 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:27:15,334 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:27:15,702 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:27:20,368 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:27:20,368 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:27:20,403 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: partial_history +2023-11-23 11:27:20,405 DEBUG SenderThread:3236 [sender.py:send():380] send: history +2023-11-23 11:27:20,405 DEBUG SenderThread:3236 [sender.py:send_request():407] send_request: summary_record +2023-11-23 11:27:20,408 INFO SenderThread:3236 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-23 11:27:20,708 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/wandb-summary.json +2023-11-23 11:27:21,409 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:27:21,712 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:27:22,717 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:27:25,582 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:27:26,410 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:27:31,228 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:27:31,558 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:27:31,783 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:27:33,776 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:27:36,229 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:27:36,777 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:27:41,843 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:27:42,024 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:27:42,230 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:27:47,231 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:27:48,472 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:27:51,916 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:27:52,850 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:27:53,473 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:27:57,851 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:27:59,286 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:28:01,980 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:28:03,465 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:28:03,777 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:28:04,286 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:28:08,779 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:28:09,751 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:28:09,751 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:28:10,038 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:28:14,021 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:28:14,752 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:28:19,897 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:28:20,102 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:28:20,368 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:28:24,897 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:28:25,531 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:28:30,166 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:28:30,473 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:28:31,280 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:28:33,777 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:28:35,779 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:28:37,088 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:28:37,088 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:28:40,247 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:28:41,330 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:28:42,089 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:28:46,332 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:28:47,630 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:28:50,306 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:28:52,018 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:28:52,631 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:28:57,020 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:28:57,750 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:29:00,376 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:29:02,089 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:29:02,751 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:29:03,778 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:29:07,722 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:29:08,337 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:29:09,442 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:29:12,723 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:29:13,338 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:29:18,341 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:29:19,156 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:29:19,506 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:29:23,342 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:29:25,574 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:29:25,574 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:29:29,318 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:29:29,568 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:29:30,575 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:29:33,778 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:29:34,779 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:29:35,891 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:29:37,628 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:29:40,148 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:29:40,892 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:29:45,150 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:29:46,116 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:29:49,724 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:29:50,544 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:29:51,117 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:29:55,546 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:29:56,863 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:29:57,776 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:30:01,123 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:30:01,864 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:30:03,781 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:30:06,644 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:30:07,339 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:30:07,847 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:30:11,645 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:30:12,359 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:30:17,033 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:30:17,789 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:30:17,924 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:30:22,034 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:30:22,791 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:30:27,103 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:30:27,999 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:30:28,014 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:30:32,104 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:30:33,781 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:30:34,608 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:30:36,042 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: partial_history +2023-11-23 11:30:36,044 DEBUG SenderThread:3236 [sender.py:send():380] send: history +2023-11-23 11:30:36,044 DEBUG SenderThread:3236 [sender.py:send_request():407] send_request: summary_record +2023-11-23 11:30:36,046 INFO SenderThread:3236 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-23 11:30:36,057 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/wandb-summary.json +2023-11-23 11:30:37,062 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:30:38,048 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:30:38,124 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:30:39,609 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:30:43,049 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:30:44,751 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:30:48,200 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:30:49,003 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:30:49,752 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:30:54,005 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:30:55,230 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:30:55,230 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:30:56,331 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:30:59,399 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:31:00,231 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:31:03,782 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:31:04,783 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:31:05,579 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:31:06,398 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:31:09,892 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:31:10,580 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:31:15,268 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:31:16,052 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:31:16,466 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:31:20,269 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:31:22,541 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:31:22,541 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:31:22,541 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:31:25,714 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:31:25,855 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:31:27,542 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:31:30,857 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:31:32,572 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:31:33,782 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:31:35,787 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:31:36,850 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:31:37,574 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:31:41,852 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:31:43,134 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:31:45,859 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:31:47,515 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:31:48,135 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:31:52,517 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:31:53,541 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:31:53,915 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:31:58,059 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:31:58,736 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:32:03,476 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:32:03,783 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:32:03,983 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:32:04,218 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:32:08,785 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:32:10,770 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:32:10,770 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:32:14,055 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:32:14,055 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:32:15,771 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:32:19,057 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:32:21,189 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:32:22,110 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:32:24,364 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:32:26,190 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:32:29,365 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:32:31,996 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:32:32,180 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:32:33,783 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:32:34,784 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:32:36,997 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:32:39,786 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:32:42,252 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:32:42,396 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:32:45,692 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:32:48,807 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:32:51,009 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:32:52,315 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:32:53,808 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:32:56,010 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:32:59,131 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:33:00,371 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:33:01,286 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:33:03,784 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:33:04,131 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:33:06,785 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:33:09,605 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:33:10,436 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:33:11,848 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:33:14,606 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:33:16,851 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:33:19,715 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:33:20,502 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:33:21,879 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:33:24,717 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:33:26,881 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:33:30,013 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:33:30,559 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:33:32,145 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:33:33,784 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:33:36,509 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:33:37,672 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:33:40,630 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:33:41,511 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:33:42,673 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:33:46,827 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:33:47,019 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: partial_history +2023-11-23 11:33:47,020 DEBUG SenderThread:3236 [sender.py:send():380] send: history +2023-11-23 11:33:47,021 DEBUG SenderThread:3236 [sender.py:send_request():407] send_request: summary_record +2023-11-23 11:33:47,022 INFO SenderThread:3236 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-23 11:33:47,676 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/wandb-summary.json +2023-11-23 11:33:48,042 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:33:48,681 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:33:50,691 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:33:51,828 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:33:53,044 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:33:56,828 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:33:58,045 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:34:01,830 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:34:03,046 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:34:03,784 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:34:06,831 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:34:08,786 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:34:11,832 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:34:13,786 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:34:16,833 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:34:18,787 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:34:21,834 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:34:23,788 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:34:26,835 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:34:28,789 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:34:31,835 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:34:33,786 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:34:34,786 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:34:36,836 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:34:39,787 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:34:41,837 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:34:44,788 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:34:46,838 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:34:49,788 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:34:51,839 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:34:54,789 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:34:56,840 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:34:59,790 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:35:01,841 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:35:03,786 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:35:05,786 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:35:06,841 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:35:10,787 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:35:11,843 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:35:15,788 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:35:16,843 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:35:20,790 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:35:21,850 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:35:25,791 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:35:26,860 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:35:30,792 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:35:31,860 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:35:33,797 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:35:35,798 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:35:36,861 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:35:40,799 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:35:41,862 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:35:45,799 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:35:46,863 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:35:50,800 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:35:51,864 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:35:55,801 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:35:56,864 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:36:00,801 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:36:01,865 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:36:03,786 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:36:06,787 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:36:06,866 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:36:11,788 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:36:11,867 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:36:16,789 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:36:16,868 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:36:21,790 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:36:21,869 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:36:26,790 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:36:26,870 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:36:31,791 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:36:31,870 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:36:33,790 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:36:36,871 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:36:37,791 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:36:41,872 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:36:42,792 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:36:46,873 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:36:47,793 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:36:51,873 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:36:52,793 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:36:56,874 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:36:57,794 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:37:01,880 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:37:02,795 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:37:03,789 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:37:06,881 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:37:08,790 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:37:11,882 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:37:13,791 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:37:16,883 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:37:18,792 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:37:21,884 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:37:23,793 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:37:26,885 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:37:28,793 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:37:31,885 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:37:33,790 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:37:34,791 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:37:36,886 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:37:39,792 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:37:41,887 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:37:44,793 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:37:46,888 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:37:49,794 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:37:51,888 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:37:54,794 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:37:56,889 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:37:59,795 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:38:01,890 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:38:03,791 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:38:05,792 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:38:06,891 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:38:10,792 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:38:11,892 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:38:15,793 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:38:16,892 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:38:20,793 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:38:21,893 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:38:25,794 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:38:26,894 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:38:30,795 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:38:31,895 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:38:33,791 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:38:36,793 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:38:36,895 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:38:41,794 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:38:41,896 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:38:46,794 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:38:46,897 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:38:51,795 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:38:51,898 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:38:56,796 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:38:56,899 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:39:01,796 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:39:01,899 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:39:03,792 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:39:06,900 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:39:07,793 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:39:11,901 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:39:12,793 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:39:16,902 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:39:17,794 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:39:21,903 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:39:22,795 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:39:26,903 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:39:27,796 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:39:31,904 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:39:32,797 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:39:33,792 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:39:36,905 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:39:38,793 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:39:41,906 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:39:43,794 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:39:46,907 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:39:48,795 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:39:51,907 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:39:53,795 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:39:56,908 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:39:58,796 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:40:01,909 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:40:03,794 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:40:04,795 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:40:06,910 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:40:09,796 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:40:11,911 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:40:14,797 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:40:16,911 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:40:19,797 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:40:21,912 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:40:24,798 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:40:26,913 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:40:29,799 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:40:31,914 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:40:33,796 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:40:34,804 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:40:36,915 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:40:39,805 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:40:41,915 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:40:44,805 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:40:46,916 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:40:49,806 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:40:51,917 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:40:54,807 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:40:56,918 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:40:59,807 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:41:01,919 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:41:03,796 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:41:05,797 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:41:06,919 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:41:10,798 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:41:11,920 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:41:15,799 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:41:16,921 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:41:20,799 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:41:21,922 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:41:25,800 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:41:26,923 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:41:30,801 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:41:31,923 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:41:33,797 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:41:36,798 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:41:36,924 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:41:41,799 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:41:41,925 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:41:46,799 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:41:46,926 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:41:51,800 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:41:51,927 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:41:56,801 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:41:56,927 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:42:01,801 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:42:01,928 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:42:03,798 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:42:06,929 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:42:07,799 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:42:11,930 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:42:12,800 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:42:16,931 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:42:17,801 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:42:21,932 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:42:22,801 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:42:26,932 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:42:27,802 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:42:31,933 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:42:32,803 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:42:33,798 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:42:36,934 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:42:38,799 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:42:41,935 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:42:43,800 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:42:46,936 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:42:48,801 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:42:51,937 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:42:53,802 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:42:56,937 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:42:58,802 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:43:01,938 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:43:03,798 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:43:04,799 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:43:06,939 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:43:09,800 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:43:11,940 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:43:14,800 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:43:16,941 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:43:19,801 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:43:21,941 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:43:24,802 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:43:26,942 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:43:29,803 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:43:31,943 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:43:33,799 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:43:35,800 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:43:36,944 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:43:40,800 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:43:41,945 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:43:45,801 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:43:46,946 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:43:47,686 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: partial_history +2023-11-23 11:43:47,689 DEBUG SenderThread:3236 [sender.py:send():380] send: history +2023-11-23 11:43:47,689 DEBUG SenderThread:3236 [sender.py:send_request():407] send_request: summary_record +2023-11-23 11:43:47,692 INFO SenderThread:3236 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-23 11:43:48,410 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/wandb-summary.json +2023-11-23 11:43:49,415 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:43:51,694 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:43:51,947 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:43:56,695 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:43:56,948 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:44:01,696 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:44:01,948 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:44:03,799 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:44:06,801 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:44:06,949 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:44:11,801 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:44:11,950 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:44:16,577 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:44:16,876 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:44:16,994 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:44:21,877 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:44:22,464 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:44:26,878 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:44:27,807 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:44:31,879 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:44:33,096 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:44:33,800 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:44:37,801 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:44:38,514 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:44:42,802 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:44:44,059 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:44:47,803 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:44:49,113 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:44:52,804 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:44:54,854 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:44:57,805 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:45:02,136 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:45:02,933 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:45:03,195 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:45:03,800 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:45:03,940 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:45:08,701 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:45:08,729 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:45:08,737 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:45:08,738 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:45:09,970 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:45:13,746 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:45:18,747 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:45:20,069 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:45:23,747 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:45:28,748 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:45:33,749 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:45:33,801 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:45:36,754 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:45:36,755 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:45:38,106 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:45:38,768 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:45:41,755 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:45:43,769 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:45:46,756 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:45:48,769 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:45:51,757 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:45:53,770 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:45:56,757 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:45:58,771 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:46:01,758 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:46:03,771 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:46:03,801 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:46:06,759 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:46:08,802 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:46:11,760 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:46:13,803 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:46:16,761 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:46:18,804 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:46:21,762 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:46:23,804 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:46:26,763 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:46:28,805 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:46:31,763 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:46:33,802 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:46:34,802 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:46:35,324 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:46:36,764 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:46:39,803 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:46:44,380 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:46:44,805 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:46:47,391 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:46:49,381 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:46:50,038 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:46:55,040 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:46:55,565 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:46:59,456 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:47:00,160 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:47:00,567 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:47:03,802 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:47:05,804 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:47:06,087 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:47:09,534 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:47:11,088 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:47:11,460 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:47:16,462 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:47:17,064 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:47:19,599 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:47:22,065 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:47:22,274 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:47:27,275 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:47:27,663 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:47:29,659 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:47:32,664 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:47:33,076 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:47:33,803 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:47:38,163 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:47:38,164 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:47:38,597 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:47:38,720 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:47:43,164 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:47:43,598 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:47:48,783 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:47:49,103 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:47:49,304 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:47:54,104 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:47:54,305 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:47:58,842 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:48:00,079 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:48:00,299 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:48:03,804 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:48:05,081 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:48:05,806 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:48:08,903 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:48:10,164 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:48:11,445 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:48:16,447 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:48:16,807 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:48:18,973 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:48:21,814 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:48:22,034 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:48:27,035 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:48:27,138 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:48:27,138 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:48:29,034 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:48:32,139 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:48:32,429 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:48:33,804 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:48:37,164 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:48:37,806 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:48:39,101 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:48:43,127 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:48:43,548 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:48:49,052 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:48:49,173 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:48:49,424 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:48:54,053 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:48:54,425 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:48:59,235 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:48:59,878 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:49:00,158 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:49:03,805 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:49:04,862 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:49:05,806 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:49:09,298 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:49:10,164 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:49:11,798 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:49:16,026 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:49:16,799 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:49:19,366 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:49:21,027 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:49:22,572 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:49:26,551 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:49:26,552 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:49:27,574 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:49:29,426 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:49:31,553 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:49:32,996 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:49:33,806 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:49:37,164 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:49:38,740 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:49:39,498 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:49:42,494 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:49:43,741 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:49:48,246 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:49:48,913 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:49:49,566 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:49:53,247 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:49:53,913 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:49:57,949 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: partial_history +2023-11-23 11:49:57,952 DEBUG SenderThread:3236 [sender.py:send():380] send: history +2023-11-23 11:49:57,956 DEBUG SenderThread:3236 [sender.py:send_request():407] send_request: summary_record +2023-11-23 11:49:57,959 INFO SenderThread:3236 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-23 11:49:58,528 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:49:58,633 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/wandb-summary.json +2023-11-23 11:49:58,961 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:49:59,640 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:50:03,529 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:50:03,806 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:50:04,807 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:50:09,165 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:50:09,713 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:50:10,775 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:50:14,494 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:50:15,777 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:50:19,787 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:50:20,326 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:50:21,727 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:50:25,336 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:50:26,729 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:50:29,861 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:50:30,387 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:50:31,742 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:50:33,807 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:50:35,388 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:50:36,808 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:50:38,919 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:50:41,046 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:50:42,377 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:50:46,898 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:50:47,378 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:50:48,996 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:50:51,899 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:50:53,190 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:50:57,532 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:50:58,750 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:50:59,055 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:51:02,533 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:51:03,751 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:51:03,807 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:51:08,123 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:51:09,131 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:51:09,478 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:51:13,124 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:51:14,479 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:51:18,315 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:51:19,189 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:51:19,729 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:51:23,378 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:51:24,730 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:51:28,643 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:51:29,247 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:51:29,800 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:51:33,644 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:51:33,807 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:51:34,808 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:51:39,131 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:51:39,315 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:51:40,535 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:51:44,133 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:51:45,536 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:51:49,391 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:51:49,929 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:51:51,130 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:51:55,641 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:51:55,642 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:51:56,132 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:51:59,453 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:52:00,642 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:52:01,218 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:52:03,808 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:52:05,643 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:52:06,809 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:52:10,704 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:52:11,810 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:52:15,705 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:52:16,811 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:52:21,384 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:52:21,987 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:52:23,579 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:52:26,385 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:52:26,988 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:52:31,787 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:52:32,114 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:52:33,636 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:52:33,808 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:52:36,835 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:52:37,810 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:52:42,451 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:52:42,811 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:52:45,817 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:52:47,459 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:52:47,913 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:52:52,915 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:52:53,362 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:52:53,362 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:52:55,879 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:52:58,363 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:52:58,616 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:53:03,397 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:53:03,617 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:53:03,809 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:53:05,944 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:53:08,398 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:53:08,810 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:53:13,548 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:53:13,815 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:53:16,012 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:53:18,549 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:53:18,995 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:53:24,235 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:53:24,562 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:53:26,070 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:53:29,236 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:53:29,570 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:53:33,199 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: partial_history +2023-11-23 11:53:33,200 DEBUG SenderThread:3236 [sender.py:send():380] send: history +2023-11-23 11:53:33,201 DEBUG SenderThread:3236 [sender.py:send_request():407] send_request: summary_record +2023-11-23 11:53:33,203 INFO SenderThread:3236 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-23 11:53:33,809 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:53:34,133 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:53:34,133 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/wandb-summary.json +2023-11-23 11:53:34,810 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:53:34,831 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:53:35,138 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:53:39,812 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:53:39,893 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:53:45,158 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:53:45,239 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:53:45,779 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:53:50,160 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:53:51,665 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:53:55,302 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:53:55,919 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:53:56,667 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:54:00,920 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:54:02,231 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:54:03,453 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:54:03,810 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:54:06,811 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:54:07,232 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:54:11,812 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:54:12,539 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:54:15,532 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:54:17,075 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:54:17,540 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:54:22,077 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:54:22,826 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:54:25,620 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:54:27,174 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:54:27,827 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:54:32,175 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:54:32,908 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:54:33,810 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:54:35,677 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:54:37,811 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:54:37,910 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:54:42,813 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:54:43,719 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:54:45,733 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:54:48,124 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:54:48,719 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:54:53,125 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:54:53,720 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:54:58,126 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:54:59,112 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:54:59,818 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:55:03,325 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:55:03,811 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:55:04,113 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:55:08,910 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:55:09,363 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:55:09,889 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:55:13,911 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:55:14,653 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:55:19,424 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:55:19,832 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:55:19,951 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:55:24,425 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:55:25,837 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:55:25,837 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:55:29,993 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:55:30,055 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:55:30,838 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:55:33,813 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:55:35,816 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:55:36,379 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:55:38,109 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:55:41,380 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:55:41,551 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:55:46,875 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:55:47,279 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:55:48,176 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:55:51,876 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:55:52,279 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:55:57,079 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:55:57,080 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:55:57,408 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:55:58,245 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:56:02,081 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:56:02,409 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:56:03,815 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:56:07,307 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:56:07,363 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:56:08,145 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:56:12,796 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:56:13,146 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:56:17,371 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:56:18,529 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:56:18,719 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:56:23,720 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:56:23,866 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:56:23,866 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:56:27,438 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:56:28,867 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:56:29,320 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:56:33,815 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:56:34,712 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:56:34,817 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:56:37,502 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:56:39,712 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:56:40,345 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:56:45,204 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:56:45,347 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:56:47,591 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:56:50,207 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:56:50,559 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:56:55,211 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: partial_history +2023-11-23 11:56:55,212 DEBUG SenderThread:3236 [sender.py:send():380] send: history +2023-11-23 11:56:55,213 DEBUG SenderThread:3236 [sender.py:send_request():407] send_request: summary_record +2023-11-23 11:56:55,215 INFO SenderThread:3236 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-23 11:56:55,646 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/wandb-summary.json +2023-11-23 11:56:55,783 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:56:56,217 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:56:56,650 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:56:57,655 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:57:00,784 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:57:01,217 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:57:03,818 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:57:05,717 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:57:06,575 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:57:06,760 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:57:11,761 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:57:13,153 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:57:13,153 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:57:15,778 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:57:17,305 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:57:18,157 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:57:22,307 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:57:23,589 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:57:25,848 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:57:27,979 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:57:28,590 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:57:32,981 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:57:33,817 DEBUG SenderThread:3236 [sender.py:send():380] send: stats +2023-11-23 11:57:34,356 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:57:35,902 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:57:38,819 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:57:39,356 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:57:43,821 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:57:44,705 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:57:45,977 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:57:49,052 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:57:49,706 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:57:54,867 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:57:55,031 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:57:56,069 INFO Thread-12 :3236 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_234140-blvl5h7j/files/output.log +2023-11-23 11:58:00,032 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: status_report +2023-11-23 11:58:00,591 DEBUG HandlerThread:3236 [handler.py:handle_request():146] handle_request: keepalive +2023-11-23 11:58:03,829 DEBUG SenderThread:3236 [sender.py:send():380] send: stats